diff --git a/src/hotspot/cpu/aarch64/aarch64_vector.ad b/src/hotspot/cpu/aarch64/aarch64_vector.ad index 7345c551d4bf1..7aacf0dfd1a9b 100644 --- a/src/hotspot/cpu/aarch64/aarch64_vector.ad +++ b/src/hotspot/cpu/aarch64/aarch64_vector.ad @@ -315,6 +315,10 @@ source %{ } } + const bool Matcher::vector_needs_load_shuffle(BasicType elem_bt, int vlen) { + return false; + } + // Assert that the given node is not a variable shift. bool assert_not_var_shift(const Node* n) { assert(!n->as_ShiftV()->is_var_shift(), "illegal variable shift"); @@ -6065,41 +6069,6 @@ instruct vtest_alltrue_sve(rFlagsReg cr, pReg src1, pReg src2, pReg ptmp) %{ ins_pipe(pipe_slow); %} -// ------------------------------ Vector shuffle ------------------------------- - -instruct loadshuffle(vReg dst, vReg src) %{ - match(Set dst (VectorLoadShuffle src)); - format %{ "loadshuffle $dst, $src" %} - ins_encode %{ - BasicType bt = Matcher::vector_element_basic_type(this); - uint length_in_bytes = Matcher::vector_length_in_bytes(this); - if (bt == T_BYTE) { - if ($dst$$FloatRegister != $src$$FloatRegister) { - if (VM_Version::use_neon_for_vector(length_in_bytes)) { - __ orr($dst$$FloatRegister, length_in_bytes == 16 ? __ T16B : __ T8B, - $src$$FloatRegister, $src$$FloatRegister); - } else { - assert(UseSVE > 0, "must be sve"); - __ sve_orr($dst$$FloatRegister, $src$$FloatRegister, $src$$FloatRegister); - } - } - } else { - if (VM_Version::use_neon_for_vector(length_in_bytes)) { - // 4S/8S, 4I, 4F - __ uxtl($dst$$FloatRegister, __ T8H, $src$$FloatRegister, __ T8B); - if (type2aelembytes(bt) == 4) { - __ uxtl($dst$$FloatRegister, __ T4S, $dst$$FloatRegister, __ T4H); - } - } else { - assert(UseSVE > 0, "must be sve"); - __ sve_vector_extend($dst$$FloatRegister, __ elemType_to_regVariant(bt), - $src$$FloatRegister, __ B); - } - } - %} - ins_pipe(pipe_slow); -%} - // ------------------------------ Vector rearrange ----------------------------- // Here is an example that rearranges a NEON vector with 4 ints: @@ -6122,6 +6091,7 @@ instruct loadshuffle(vReg dst, vReg src) %{ // need to lookup 2/4 bytes as a group. For VectorRearrange long, we use bsl // to implement rearrange. +// Maybe move the shuffle preparation to VectorLoadShuffle instruct rearrange_HS_neon(vReg dst, vReg src, vReg shuffle, vReg tmp1, vReg tmp2) %{ predicate(UseSVE == 0 && (Matcher::vector_element_basic_type(n) == T_SHORT || diff --git a/src/hotspot/cpu/aarch64/aarch64_vector_ad.m4 b/src/hotspot/cpu/aarch64/aarch64_vector_ad.m4 index a13f5fdeb4466..1b5253e46e07f 100644 --- a/src/hotspot/cpu/aarch64/aarch64_vector_ad.m4 +++ b/src/hotspot/cpu/aarch64/aarch64_vector_ad.m4 @@ -305,6 +305,10 @@ source %{ } } + const bool Matcher::vector_needs_load_shuffle(BasicType elem_bt, int vlen) { + return false; + } + // Assert that the given node is not a variable shift. bool assert_not_var_shift(const Node* n) { assert(!n->as_ShiftV()->is_var_shift(), "illegal variable shift"); @@ -4418,41 +4422,6 @@ instruct vtest_alltrue_sve(rFlagsReg cr, pReg src1, pReg src2, pReg ptmp) %{ ins_pipe(pipe_slow); %} -// ------------------------------ Vector shuffle ------------------------------- - -instruct loadshuffle(vReg dst, vReg src) %{ - match(Set dst (VectorLoadShuffle src)); - format %{ "loadshuffle $dst, $src" %} - ins_encode %{ - BasicType bt = Matcher::vector_element_basic_type(this); - uint length_in_bytes = Matcher::vector_length_in_bytes(this); - if (bt == T_BYTE) { - if ($dst$$FloatRegister != $src$$FloatRegister) { - if (VM_Version::use_neon_for_vector(length_in_bytes)) { - __ orr($dst$$FloatRegister, length_in_bytes == 16 ? __ T16B : __ T8B, - $src$$FloatRegister, $src$$FloatRegister); - } else { - assert(UseSVE > 0, "must be sve"); - __ sve_orr($dst$$FloatRegister, $src$$FloatRegister, $src$$FloatRegister); - } - } - } else { - if (VM_Version::use_neon_for_vector(length_in_bytes)) { - // 4S/8S, 4I, 4F - __ uxtl($dst$$FloatRegister, __ T8H, $src$$FloatRegister, __ T8B); - if (type2aelembytes(bt) == 4) { - __ uxtl($dst$$FloatRegister, __ T4S, $dst$$FloatRegister, __ T4H); - } - } else { - assert(UseSVE > 0, "must be sve"); - __ sve_vector_extend($dst$$FloatRegister, __ elemType_to_regVariant(bt), - $src$$FloatRegister, __ B); - } - } - %} - ins_pipe(pipe_slow); -%} - // ------------------------------ Vector rearrange ----------------------------- // Here is an example that rearranges a NEON vector with 4 ints: @@ -4475,6 +4444,7 @@ instruct loadshuffle(vReg dst, vReg src) %{ // need to lookup 2/4 bytes as a group. For VectorRearrange long, we use bsl // to implement rearrange. +// Maybe move the shuffle preparation to VectorLoadShuffle instruct rearrange_HS_neon(vReg dst, vReg src, vReg shuffle, vReg tmp1, vReg tmp2) %{ predicate(UseSVE == 0 && (Matcher::vector_element_basic_type(n) == T_SHORT || diff --git a/src/hotspot/cpu/arm/arm.ad b/src/hotspot/cpu/arm/arm.ad index 611d90692a75b..581658f5b78a8 100644 --- a/src/hotspot/cpu/arm/arm.ad +++ b/src/hotspot/cpu/arm/arm.ad @@ -1025,6 +1025,10 @@ const bool Matcher::vector_needs_partial_operations(Node* node, const TypeVect* return false; } +const bool Matcher::vector_needs_load_shuffle(BasicType elem_bt, int vlen) { + return false; +} + const RegMask* Matcher::predicate_reg_mask(void) { return NULL; } diff --git a/src/hotspot/cpu/ppc/ppc.ad b/src/hotspot/cpu/ppc/ppc.ad index d489118c4a132..bc7f6a3afe65c 100644 --- a/src/hotspot/cpu/ppc/ppc.ad +++ b/src/hotspot/cpu/ppc/ppc.ad @@ -2189,6 +2189,10 @@ const bool Matcher::vector_needs_partial_operations(Node* node, const TypeVect* return false; } +const bool Matcher::vector_needs_load_shuffle(BasicType elem_bt, int vlen) { + return false; +} + const RegMask* Matcher::predicate_reg_mask(void) { return NULL; } diff --git a/src/hotspot/cpu/riscv/riscv.ad b/src/hotspot/cpu/riscv/riscv.ad index fe20a5c6054c9..8696bef815a34 100644 --- a/src/hotspot/cpu/riscv/riscv.ad +++ b/src/hotspot/cpu/riscv/riscv.ad @@ -1872,6 +1872,10 @@ const bool Matcher::vector_needs_partial_operations(Node* node, const TypeVect* return false; } +const bool Matcher::vector_needs_load_shuffle(BasicType elem_bt, int vlen) { + return false; +} + const RegMask* Matcher::predicate_reg_mask(void) { return NULL; } diff --git a/src/hotspot/cpu/s390/s390.ad b/src/hotspot/cpu/s390/s390.ad index 7f32d5454c808..5f8003a04c1e7 100644 --- a/src/hotspot/cpu/s390/s390.ad +++ b/src/hotspot/cpu/s390/s390.ad @@ -1529,6 +1529,10 @@ const bool Matcher::vector_needs_partial_operations(Node* node, const TypeVect* return false; } +const bool Matcher::vector_needs_load_shuffle(BasicType elem_bt, int vlen) { + return false; +} + const RegMask* Matcher::predicate_reg_mask(void) { return NULL; } diff --git a/src/hotspot/cpu/x86/x86.ad b/src/hotspot/cpu/x86/x86.ad index 56e754965d375..4fe99953efef1 100644 --- a/src/hotspot/cpu/x86/x86.ad +++ b/src/hotspot/cpu/x86/x86.ad @@ -2171,6 +2171,19 @@ const bool Matcher::vector_needs_partial_operations(Node* node, const TypeVect* return false; } +// Return true if Vector::rearrange needs preparation of the shuffle argument +const bool Matcher::vector_needs_load_shuffle(BasicType elem_bt, int vlen) { + switch (elem_bt) { + case T_BYTE: return false; + case T_SHORT: return !VM_Version::supports_avx512bw(); + case T_INT: return !VM_Version::supports_avx(); + case T_LONG: return vlen < 8 && !VM_Version::supports_avx512vl(); + default: + ShouldNotReachHere(); + return false; + } +} + MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* generic_opnd, uint ideal_reg, bool is_temp) { assert(Matcher::is_generic_vector(generic_opnd), "not generic"); bool legacy = (generic_opnd->opcode() == LEGVEC); @@ -8406,17 +8419,6 @@ instruct VectorPopulateLIndex(vec dst, rRegL src1, immI_1 src2, vec vtmp) %{ //-------------------------------- Rearrange ---------------------------------- // LoadShuffle/Rearrange for Byte - -instruct loadShuffleB(vec dst) %{ - predicate(Matcher::vector_element_basic_type(n) == T_BYTE); - match(Set dst (VectorLoadShuffle dst)); - format %{ "vector_load_shuffle $dst, $dst" %} - ins_encode %{ - // empty - %} - ins_pipe( pipe_slow ); -%} - instruct rearrangeB(vec dst, vec shuffle) %{ predicate(Matcher::vector_element_basic_type(n) == T_BYTE && Matcher::vector_length(n) < 32); @@ -8483,7 +8485,7 @@ instruct rearrangeB_evex_vbmi(vec dst, vec src, vec shuffle) %{ instruct loadShuffleS(vec dst, vec src, vec vtmp) %{ predicate(Matcher::vector_element_basic_type(n) == T_SHORT && - Matcher::vector_length(n) <= 16 && !VM_Version::supports_avx512bw()); // NB! aligned with rearrangeS + !VM_Version::supports_avx512bw()); match(Set dst (VectorLoadShuffle src)); effect(TEMP dst, TEMP vtmp); format %{ "vector_load_shuffle $dst, $src\t! using $vtmp as TEMP" %} @@ -8494,7 +8496,7 @@ instruct loadShuffleS(vec dst, vec src, vec vtmp) %{ if (UseAVX == 0) { assert(vlen_in_bytes <= 16, "required"); // Multiply each shuffle by two to get byte index - __ pmovzxbw($vtmp$$XMMRegister, $src$$XMMRegister); + __ movdqu($vtmp$$XMMRegister, $src$$XMMRegister); __ psllw($vtmp$$XMMRegister, 1); // Duplicate to create 2 copies of byte index @@ -8509,8 +8511,7 @@ instruct loadShuffleS(vec dst, vec src, vec vtmp) %{ assert(UseAVX > 1 || vlen_in_bytes <= 16, "required"); int vlen_enc = vector_length_encoding(this); // Multiply each shuffle by two to get byte index - __ vpmovzxbw($vtmp$$XMMRegister, $src$$XMMRegister, vlen_enc); - __ vpsllw($vtmp$$XMMRegister, $vtmp$$XMMRegister, 1, vlen_enc); + __ vpsllw($vtmp$$XMMRegister, $src$$XMMRegister, 1, vlen_enc); // Duplicate to create 2 copies of byte index __ vpsllw($dst$$XMMRegister, $vtmp$$XMMRegister, 8, vlen_enc); @@ -8557,21 +8558,6 @@ instruct rearrangeS_avx(legVec dst, legVec src, vec shuffle, legVec vtmp1, legVe ins_pipe( pipe_slow ); %} -instruct loadShuffleS_evex(vec dst, vec src) %{ - predicate(Matcher::vector_element_basic_type(n) == T_SHORT && - VM_Version::supports_avx512bw()); - match(Set dst (VectorLoadShuffle src)); - format %{ "vector_load_shuffle $dst, $src" %} - ins_encode %{ - int vlen_enc = vector_length_encoding(this); - if (!VM_Version::supports_avx512vl()) { - vlen_enc = Assembler::AVX_512bit; - } - __ vpmovzxbw($dst$$XMMRegister, $src$$XMMRegister, vlen_enc); - %} - ins_pipe( pipe_slow ); -%} - instruct rearrangeS_evex(vec dst, vec src, vec shuffle) %{ predicate(Matcher::vector_element_basic_type(n) == T_SHORT && VM_Version::supports_avx512bw()); @@ -8602,7 +8588,7 @@ instruct loadShuffleI(vec dst, vec src, vec vtmp) %{ // only byte shuffle instruction available on these platforms // Duplicate and multiply each shuffle by 4 - __ pmovzxbd($vtmp$$XMMRegister, $src$$XMMRegister); + __ movdqu($vtmp$$XMMRegister, $src$$XMMRegister); __ pshuflw($vtmp$$XMMRegister, $vtmp$$XMMRegister, 0xA0); __ pshufhw($vtmp$$XMMRegister, $vtmp$$XMMRegister, 0xA0); __ psllw($vtmp$$XMMRegister, 2); @@ -8631,18 +8617,6 @@ instruct rearrangeI(vec dst, vec shuffle) %{ ins_pipe( pipe_slow ); %} -instruct loadShuffleI_avx(vec dst, vec src) %{ - predicate((Matcher::vector_element_basic_type(n) == T_INT || Matcher::vector_element_basic_type(n) == T_FLOAT) && - UseAVX > 0); - match(Set dst (VectorLoadShuffle src)); - format %{ "vector_load_shuffle $dst, $src" %} - ins_encode %{ - int vlen_enc = vector_length_encoding(this); - __ vpmovzxbd($dst$$XMMRegister, $src$$XMMRegister, vlen_enc); - %} - ins_pipe( pipe_slow ); -%} - instruct rearrangeI_avx(vec dst, vec src, vec shuffle) %{ predicate((Matcher::vector_element_basic_type(n) == T_INT || Matcher::vector_element_basic_type(n) == T_FLOAT) && UseAVX > 0); @@ -8672,8 +8646,7 @@ instruct loadShuffleL(vec dst, vec src, vec vtmp) %{ // only double word shuffle instruction available on these platforms // Multiply each shuffle by two to get double word index - __ vpmovzxbq($vtmp$$XMMRegister, $src$$XMMRegister, vlen_enc); - __ vpsllq($vtmp$$XMMRegister, $vtmp$$XMMRegister, 1, vlen_enc); + __ vpsllq($vtmp$$XMMRegister, $src$$XMMRegister, 1, vlen_enc); // Duplicate each double word shuffle __ vpsllq($dst$$XMMRegister, $vtmp$$XMMRegister, 32, vlen_enc); @@ -8699,20 +8672,6 @@ instruct rearrangeL(vec dst, vec src, vec shuffle) %{ ins_pipe( pipe_slow ); %} -instruct loadShuffleL_evex(vec dst, vec src) %{ - predicate(is_double_word_type(Matcher::vector_element_basic_type(n)) && // T_LONG, T_DOUBLE - (Matcher::vector_length(n) == 8 || VM_Version::supports_avx512vl())); - match(Set dst (VectorLoadShuffle src)); - format %{ "vector_load_shuffle $dst, $src" %} - ins_encode %{ - assert(UseAVX > 2, "required"); - - int vlen_enc = vector_length_encoding(this); - __ vpmovzxbq($dst$$XMMRegister, $src$$XMMRegister, vlen_enc); - %} - ins_pipe( pipe_slow ); -%} - instruct rearrangeL_evex(vec dst, vec src, vec shuffle) %{ predicate(is_double_word_type(Matcher::vector_element_basic_type(n)) && // T_LONG, T_DOUBLE (Matcher::vector_length(n) == 8 || VM_Version::supports_avx512vl())); diff --git a/src/hotspot/share/classfile/vmIntrinsics.hpp b/src/hotspot/share/classfile/vmIntrinsics.hpp index f2f1e2f9caca9..96c9f78577f84 100644 --- a/src/hotspot/share/classfile/vmIntrinsics.hpp +++ b/src/hotspot/share/classfile/vmIntrinsics.hpp @@ -963,24 +963,6 @@ class methodHandle; "Ljdk/internal/vm/vector/VectorSupport$VectorPayload;") \ do_name(vector_frombits_coerced_name, "fromBitsCoerced") \ \ - do_intrinsic(_VectorShuffleIota, jdk_internal_vm_vector_VectorSupport, vector_shuffle_step_iota_name, vector_shuffle_step_iota_sig, F_S) \ - do_signature(vector_shuffle_step_iota_sig, "(Ljava/lang/Class;" \ - "Ljava/lang/Class;" \ - "Ljdk/internal/vm/vector/VectorSupport$VectorSpecies;" \ - "IIII" \ - "Ljdk/internal/vm/vector/VectorSupport$ShuffleIotaOperation;)" \ - "Ljdk/internal/vm/vector/VectorSupport$VectorShuffle;") \ - do_name(vector_shuffle_step_iota_name, "shuffleIota") \ - \ - do_intrinsic(_VectorShuffleToVector, jdk_internal_vm_vector_VectorSupport, vector_shuffle_to_vector_name, vector_shuffle_to_vector_sig, F_S) \ - do_signature(vector_shuffle_to_vector_sig, "(Ljava/lang/Class;" \ - "Ljava/lang/Class;" \ - "Ljava/lang/Class;" \ - "Ljdk/internal/vm/vector/VectorSupport$VectorShuffle;" \ - "ILjdk/internal/vm/vector/VectorSupport$ShuffleToVectorOperation;)" \ - "Ljdk/internal/vm/vector/VectorSupport$Vector;") \ - do_name(vector_shuffle_to_vector_name, "shuffleToVector") \ - \ do_intrinsic(_VectorLoadOp, jdk_internal_vm_vector_VectorSupport, vector_load_op_name, vector_load_op_sig, F_S) \ do_signature(vector_load_op_sig, "(Ljava/lang/Class;" \ "Ljava/lang/Class;" \ diff --git a/src/hotspot/share/opto/c2compiler.cpp b/src/hotspot/share/opto/c2compiler.cpp index 0af30550e62ed..560937d9fa8f4 100644 --- a/src/hotspot/share/opto/c2compiler.cpp +++ b/src/hotspot/share/opto/c2compiler.cpp @@ -754,8 +754,6 @@ bool C2Compiler::is_intrinsic_supported(const methodHandle& method) { case vmIntrinsics::_VectorBinaryOp: case vmIntrinsics::_VectorTernaryOp: case vmIntrinsics::_VectorFromBitsCoerced: - case vmIntrinsics::_VectorShuffleIota: - case vmIntrinsics::_VectorShuffleToVector: case vmIntrinsics::_VectorLoadOp: case vmIntrinsics::_VectorLoadMaskedOp: case vmIntrinsics::_VectorStoreOp: diff --git a/src/hotspot/share/opto/graphKit.hpp b/src/hotspot/share/opto/graphKit.hpp index fe731fc35cad6..ca35d1470c239 100644 --- a/src/hotspot/share/opto/graphKit.hpp +++ b/src/hotspot/share/opto/graphKit.hpp @@ -904,7 +904,7 @@ class GraphKit : public Phase { // Vector API support (implemented in vectorIntrinsics.cpp) Node* box_vector(Node* in, const TypeInstPtr* vbox_type, BasicType elem_bt, int num_elem, bool deoptimize_on_exception = false); - Node* unbox_vector(Node* in, const TypeInstPtr* vbox_type, BasicType elem_bt, int num_elem, bool shuffle_to_vector = false); + Node* unbox_vector(Node* in, const TypeInstPtr* vbox_type, BasicType elem_bt, int num_elem); Node* vector_shift_count(Node* cnt, int shift_op, BasicType bt, int num_elem); }; diff --git a/src/hotspot/share/opto/library_call.cpp b/src/hotspot/share/opto/library_call.cpp index 5f8d8e0599a96..d19e0a3ed65ff 100644 --- a/src/hotspot/share/opto/library_call.cpp +++ b/src/hotspot/share/opto/library_call.cpp @@ -694,12 +694,8 @@ bool LibraryCallKit::try_to_inline(int predicate) { return inline_vector_nary_operation(3); case vmIntrinsics::_VectorFromBitsCoerced: return inline_vector_frombits_coerced(); - case vmIntrinsics::_VectorShuffleIota: - return inline_vector_shuffle_iota(); case vmIntrinsics::_VectorMaskOp: return inline_vector_mask_operation(); - case vmIntrinsics::_VectorShuffleToVector: - return inline_vector_shuffle_to_vector(); case vmIntrinsics::_VectorLoadOp: return inline_vector_mem_operation(/*is_store=*/false); case vmIntrinsics::_VectorLoadMaskedOp: diff --git a/src/hotspot/share/opto/library_call.hpp b/src/hotspot/share/opto/library_call.hpp index 457b056918218..ac9e8ab6a4725 100644 --- a/src/hotspot/share/opto/library_call.hpp +++ b/src/hotspot/share/opto/library_call.hpp @@ -345,8 +345,6 @@ class LibraryCallKit : public GraphKit { // Vector API support bool inline_vector_nary_operation(int n); bool inline_vector_frombits_coerced(); - bool inline_vector_shuffle_to_vector(); - bool inline_vector_shuffle_iota(); bool inline_vector_mask_operation(); bool inline_vector_mem_operation(bool is_store); bool inline_vector_mem_masked_operation(bool is_store); diff --git a/src/hotspot/share/opto/matcher.hpp b/src/hotspot/share/opto/matcher.hpp index 9c10f2a666bd4..cd8b068ca1f84 100644 --- a/src/hotspot/share/opto/matcher.hpp +++ b/src/hotspot/share/opto/matcher.hpp @@ -338,6 +338,8 @@ class Matcher : public PhaseTransform { static const bool vector_needs_partial_operations(Node* node, const TypeVect* vt); + static const bool vector_needs_load_shuffle(BasicType elem_bt, int vlen); + static const RegMask* predicate_reg_mask(void); static const TypeVectMask* predicate_reg_type(const Type* elemTy, int length); diff --git a/src/hotspot/share/opto/vector.cpp b/src/hotspot/share/opto/vector.cpp index 55fce80af072c..97b44e37f0641 100644 --- a/src/hotspot/share/opto/vector.cpp +++ b/src/hotspot/share/opto/vector.cpp @@ -36,11 +36,6 @@ static bool is_vector_mask(ciKlass* klass) { return klass->is_subclass_of(ciEnv::current()->vector_VectorMask_klass()); } -static bool is_vector_shuffle(ciKlass* klass) { - return klass->is_subclass_of(ciEnv::current()->vector_VectorShuffle_klass()); -} - - void PhaseVector::optimize_vector_boxes() { Compile::TracePhase tp("vector_elimination", &timers[_t_vector_elimination]); @@ -438,8 +433,6 @@ void PhaseVector::expand_vunbox_node(VectorUnboxNode* vec_unbox) { if (is_vector_mask(from_kls)) { bt = T_BOOLEAN; - } else if (is_vector_shuffle(from_kls)) { - bt = T_BYTE; } ciField* field = ciEnv::current()->vector_VectorPayload_klass()->get_field_by_name(ciSymbols::payload_name(), @@ -484,9 +477,6 @@ void PhaseVector::expand_vunbox_node(VectorUnboxNode* vec_unbox) { if (is_vector_mask(from_kls)) { vec_val_load = gvn.transform(new VectorLoadMaskNode(vec_val_load, TypeVect::makemask(masktype, num_elem))); - } else if (is_vector_shuffle(from_kls) && !vec_unbox->is_shuffle_to_vector()) { - assert(vec_unbox->bottom_type()->is_vect()->element_basic_type() == masktype, "expect shuffle type consistency"); - vec_val_load = gvn.transform(new VectorLoadShuffleNode(vec_val_load, TypeVect::make(masktype, num_elem))); } gvn.hash_delete(vec_unbox); diff --git a/src/hotspot/share/opto/vectorIntrinsics.cpp b/src/hotspot/share/opto/vectorIntrinsics.cpp index ae0315c61c767..734120104a674 100644 --- a/src/hotspot/share/opto/vectorIntrinsics.cpp +++ b/src/hotspot/share/opto/vectorIntrinsics.cpp @@ -63,10 +63,6 @@ static bool is_vector_mask(ciKlass* klass) { return klass->is_subclass_of(ciEnv::current()->vector_VectorMask_klass()); } -static bool is_vector_shuffle(ciKlass* klass) { - return klass->is_subclass_of(ciEnv::current()->vector_VectorShuffle_klass()); -} - bool LibraryCallKit::arch_supports_vector_rotate(int opc, int num_elem, BasicType elem_bt, VectorMaskUseType mask_use_type, bool has_scalar_args) { bool is_supported = true; @@ -160,7 +156,7 @@ Node* GraphKit::box_vector(Node* vector, const TypeInstPtr* vbox_type, BasicType return gvn().transform(vbox); } -Node* GraphKit::unbox_vector(Node* v, const TypeInstPtr* vbox_type, BasicType elem_bt, int num_elem, bool shuffle_to_vector) { +Node* GraphKit::unbox_vector(Node* v, const TypeInstPtr* vbox_type, BasicType elem_bt, int num_elem) { assert(EnableVectorSupport, ""); const TypeInstPtr* vbox_type_v = gvn().type(v)->is_instptr(); if (vbox_type->instance_klass() != vbox_type_v->instance_klass()) { @@ -171,7 +167,7 @@ Node* GraphKit::unbox_vector(Node* v, const TypeInstPtr* vbox_type, BasicType el } assert(check_vbox(vbox_type), ""); const TypeVect* vt = TypeVect::make(elem_bt, num_elem, is_vector_mask(vbox_type->instance_klass())); - Node* unbox = gvn().transform(new VectorUnboxNode(C, vt, v, merged_memory(), shuffle_to_vector)); + Node* unbox = gvn().transform(new VectorUnboxNode(C, vt, v, merged_memory())); return unbox; } @@ -582,103 +578,6 @@ bool LibraryCallKit::inline_vector_nary_operation(int n) { return true; } -// , E> -// Sh ShuffleIota(Class E, Class shuffleClass, Vector.Species s, int length, -// int start, int step, int wrap, ShuffleIotaOperation defaultImpl) -bool LibraryCallKit::inline_vector_shuffle_iota() { - const TypeInstPtr* shuffle_klass = gvn().type(argument(1))->isa_instptr(); - const TypeInt* vlen = gvn().type(argument(3))->isa_int(); - const TypeInt* start_val = gvn().type(argument(4))->isa_int(); - const TypeInt* step_val = gvn().type(argument(5))->isa_int(); - const TypeInt* wrap = gvn().type(argument(6))->isa_int(); - - Node* start = argument(4); - Node* step = argument(5); - - if (shuffle_klass == nullptr || vlen == nullptr || start_val == nullptr || step_val == nullptr || wrap == nullptr) { - return false; // dead code - } - if (!vlen->is_con() || !is_power_of_2(vlen->get_con()) || - shuffle_klass->const_oop() == nullptr || !wrap->is_con()) { - return false; // not enough info for intrinsification - } - if (!is_klass_initialized(shuffle_klass)) { - if (C->print_intrinsics()) { - tty->print_cr(" ** klass argument not initialized"); - } - return false; - } - - int do_wrap = wrap->get_con(); - int num_elem = vlen->get_con(); - BasicType elem_bt = T_BYTE; - - if (!arch_supports_vector(VectorNode::replicate_opcode(elem_bt), num_elem, elem_bt, VecMaskNotUsed)) { - return false; - } - if (!arch_supports_vector(Op_AddVB, num_elem, elem_bt, VecMaskNotUsed)) { - return false; - } - if (!arch_supports_vector(Op_AndV, num_elem, elem_bt, VecMaskNotUsed)) { - return false; - } - if (!arch_supports_vector(Op_VectorLoadConst, num_elem, elem_bt, VecMaskNotUsed)) { - return false; - } - if (!arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad)) { - return false; - } - if (!arch_supports_vector(Op_VectorMaskCmp, num_elem, elem_bt, VecMaskUseStore)) { - return false; - } - - const Type * type_bt = Type::get_const_basic_type(elem_bt); - const TypeVect * vt = TypeVect::make(type_bt, num_elem); - - Node* res = gvn().transform(new VectorLoadConstNode(gvn().makecon(TypeInt::ZERO), vt)); - - if(!step_val->is_con() || !is_power_of_2(step_val->get_con())) { - Node* bcast_step = gvn().transform(VectorNode::scalar2vector(step, num_elem, type_bt)); - res = gvn().transform(VectorNode::make(Op_MulI, res, bcast_step, num_elem, elem_bt)); - } else if (step_val->get_con() > 1) { - Node* cnt = gvn().makecon(TypeInt::make(log2i_exact(step_val->get_con()))); - Node* shift_cnt = vector_shift_count(cnt, Op_LShiftI, elem_bt, num_elem); - res = gvn().transform(VectorNode::make(Op_LShiftVB, res, shift_cnt, vt)); - } - - if (!start_val->is_con() || start_val->get_con() != 0) { - Node* bcast_start = gvn().transform(VectorNode::scalar2vector(start, num_elem, type_bt)); - res = gvn().transform(VectorNode::make(Op_AddI, res, bcast_start, num_elem, elem_bt)); - } - - Node * mod_val = gvn().makecon(TypeInt::make(num_elem-1)); - Node * bcast_mod = gvn().transform(VectorNode::scalar2vector(mod_val, num_elem, type_bt)); - if(do_wrap) { - // Wrap the indices greater than lane count. - res = gvn().transform(VectorNode::make(Op_AndI, res, bcast_mod, num_elem, elem_bt)); - } else { - ConINode* pred_node = (ConINode*)gvn().makecon(TypeInt::make(BoolTest::ge)); - Node * lane_cnt = gvn().makecon(TypeInt::make(num_elem)); - Node * bcast_lane_cnt = gvn().transform(VectorNode::scalar2vector(lane_cnt, num_elem, type_bt)); - const TypeVect* vmask_type = TypeVect::makemask(elem_bt, num_elem); - Node* mask = gvn().transform(new VectorMaskCmpNode(BoolTest::ge, bcast_lane_cnt, res, pred_node, vmask_type)); - - // Make the indices greater than lane count as -ve values. This matches the java side implementation. - res = gvn().transform(VectorNode::make(Op_AndI, res, bcast_mod, num_elem, elem_bt)); - Node * biased_val = gvn().transform(VectorNode::make(Op_SubI, res, bcast_lane_cnt, num_elem, elem_bt)); - res = gvn().transform(new VectorBlendNode(biased_val, res, mask)); - } - - ciKlass* sbox_klass = shuffle_klass->const_oop()->as_instance()->java_lang_Class_klass(); - const TypeInstPtr* shuffle_box_type = TypeInstPtr::make_exact(TypePtr::NotNull, sbox_klass); - - // Wrap it up in VectorBox to keep object type information. - res = box_vector(res, shuffle_box_type, elem_bt, num_elem); - set_result(res); - C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt)))); - return true; -} - // // long maskReductionCoerced(int oper, Class maskClass, Class elemClass, // int length, M m, VectorMaskOp defaultImpl) @@ -716,7 +615,7 @@ bool LibraryCallKit::inline_vector_mask_operation() { const Type* elem_ty = Type::get_const_basic_type(elem_bt); ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass(); const TypeInstPtr* mask_box_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass); - Node* mask_vec = unbox_vector(mask, mask_box_type, elem_bt, num_elem, true); + Node* mask_vec = unbox_vector(mask, mask_box_type, elem_bt, num_elem); if (mask_vec == nullptr) { if (C->print_intrinsics()) { tty->print_cr(" ** unbox failed mask=%s", @@ -739,71 +638,6 @@ bool LibraryCallKit::inline_vector_mask_operation() { return true; } -// public static -// , -// E> -// V shuffleToVector(Class> vclass, Class elementType, -// Class shuffleClass, Sh s, int length, -// ShuffleToVectorOperation defaultImpl) -bool LibraryCallKit::inline_vector_shuffle_to_vector() { - const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr(); - const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr(); - const TypeInstPtr* shuffle_klass = gvn().type(argument(2))->isa_instptr(); - Node* shuffle = argument(3); - const TypeInt* vlen = gvn().type(argument(4))->isa_int(); - - if (vector_klass == nullptr || elem_klass == nullptr || shuffle_klass == nullptr || shuffle->is_top() || vlen == nullptr) { - return false; // dead code - } - if (!vlen->is_con() || vector_klass->const_oop() == nullptr || shuffle_klass->const_oop() == nullptr) { - return false; // not enough info for intrinsification - } - if (!is_klass_initialized(shuffle_klass) || !is_klass_initialized(vector_klass) ) { - if (C->print_intrinsics()) { - tty->print_cr(" ** klass argument not initialized"); - } - return false; - } - - int num_elem = vlen->get_con(); - ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type(); - BasicType elem_bt = elem_type->basic_type(); - - if (num_elem < 4) { - return false; - } - - int cast_vopc = VectorCastNode::opcode(-1, T_BYTE); // from shuffle of type T_BYTE - // Make sure that cast is implemented to particular type/size combination. - if (!arch_supports_vector(cast_vopc, num_elem, elem_bt, VecMaskNotUsed)) { - if (C->print_intrinsics()) { - tty->print_cr(" ** not supported: arity=1 op=cast#%d/3 vlen2=%d etype2=%s", - cast_vopc, num_elem, type2name(elem_bt)); - } - return false; - } - - ciKlass* sbox_klass = shuffle_klass->const_oop()->as_instance()->java_lang_Class_klass(); - const TypeInstPtr* shuffle_box_type = TypeInstPtr::make_exact(TypePtr::NotNull, sbox_klass); - - // Unbox shuffle with true flag to indicate its load shuffle to vector - // shuffle is a byte array - Node* shuffle_vec = unbox_vector(shuffle, shuffle_box_type, T_BYTE, num_elem, true); - - // cast byte to target element type - shuffle_vec = gvn().transform(VectorCastNode::make(cast_vopc, shuffle_vec, elem_bt, num_elem)); - - ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass(); - const TypeInstPtr* vec_box_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass); - - // Box vector - Node* res = box_vector(shuffle_vec, vec_box_type, elem_bt, num_elem); - set_result(res); - C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt)))); - return true; -} - // public static // , @@ -2058,14 +1892,22 @@ bool LibraryCallKit::inline_vector_rearrange() { } return false; // should be primitive type } + BasicType elem_bt = elem_type->basic_type(); BasicType shuffle_bt = elem_bt; + if (shuffle_bt == T_FLOAT) { + shuffle_bt = T_INT; + } else if (shuffle_bt == T_DOUBLE) { + shuffle_bt = T_LONG; + } + int num_elem = vlen->get_con(); + bool need_load_shuffle = Matcher::vector_needs_load_shuffle(shuffle_bt, num_elem); - if (!arch_supports_vector(Op_VectorLoadShuffle, num_elem, elem_bt, VecMaskNotUsed)) { + if (need_load_shuffle && !arch_supports_vector(Op_VectorLoadShuffle, num_elem, shuffle_bt, VecMaskNotUsed)) { if (C->print_intrinsics()) { tty->print_cr(" ** not supported: arity=0 op=load/shuffle vlen=%d etype=%s ismask=no", - num_elem, type2name(elem_bt)); + num_elem, type2name(shuffle_bt)); } return false; // not supported } @@ -2102,6 +1944,8 @@ bool LibraryCallKit::inline_vector_rearrange() { Node* v1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem); Node* shuffle = unbox_vector(argument(6), shbox_type, shuffle_bt, num_elem); + const TypeVect* vt = TypeVect::make(elem_bt, num_elem); + const TypeVect* st = TypeVect::make(shuffle_bt, num_elem); if (v1 == nullptr || shuffle == nullptr) { return false; // operand unboxing failed @@ -2121,13 +1965,16 @@ bool LibraryCallKit::inline_vector_rearrange() { } } + if (need_load_shuffle) { + shuffle = gvn().transform(new VectorLoadShuffleNode(shuffle, st)); + } + Node* rearrange = new VectorRearrangeNode(v1, shuffle); if (is_masked_op) { if (use_predicate) { rearrange->add_req(mask); rearrange->add_flag(Node::Flag_is_predicated_vector); } else { - const TypeVect* vt = v1->bottom_type()->is_vect(); rearrange = gvn().transform(rearrange); Node* zero = gvn().makecon(Type::get_zero_type(elem_bt)); Node* zerovec = gvn().transform(VectorNode::scalar2vector(zero, num_elem, Type::get_const_basic_type(elem_bt))); @@ -2421,9 +2268,7 @@ bool LibraryCallKit::inline_vector_convert() { ciKlass* vbox_klass_from = vector_klass_from->const_oop()->as_instance()->java_lang_Class_klass(); ciKlass* vbox_klass_to = vector_klass_to->const_oop()->as_instance()->java_lang_Class_klass(); - if (is_vector_shuffle(vbox_klass_from)) { - return false; // vector shuffles aren't supported - } + bool is_mask = is_vector_mask(vbox_klass_from); ciType* elem_type_from = elem_klass_from->const_oop()->as_instance()->java_mirror_type(); diff --git a/src/hotspot/share/opto/vectornode.cpp b/src/hotspot/share/opto/vectornode.cpp index c51958be06bd7..7e4a16c6d3b74 100644 --- a/src/hotspot/share/opto/vectornode.cpp +++ b/src/hotspot/share/opto/vectornode.cpp @@ -1644,19 +1644,13 @@ Node* VectorUnboxNode::Ideal(PhaseGVN* phase, bool can_reshape) { if (in_vt->length() == out_vt->length()) { Node* value = vbox->in(VectorBoxNode::Value); - bool is_vector_mask = vbox_klass->is_subclass_of(ciEnv::current()->vector_VectorMask_klass()); - bool is_vector_shuffle = vbox_klass->is_subclass_of(ciEnv::current()->vector_VectorShuffle_klass()); + bool is_vector_mask = vbox_klass->is_subclass_of(ciEnv::current()->vector_VectorMask_klass()); if (is_vector_mask) { // VectorUnbox (VectorBox vmask) ==> VectorMaskCast vmask const TypeVect* vmask_type = TypeVect::makemask(out_vt->element_basic_type(), out_vt->length()); return new VectorMaskCastNode(value, vmask_type); - } else if (is_vector_shuffle) { - if (!is_shuffle_to_vector()) { - // VectorUnbox (VectorBox vshuffle) ==> VectorLoadShuffle vshuffle - return new VectorLoadShuffleNode(value, out_vt); - } } else { - // Vector type mismatch is only supported for masks and shuffles, but sometimes it happens in pathological cases. + // Vector type mismatch is only supported for masks, but sometimes it happens in pathological cases. } } else { // Vector length mismatch. diff --git a/src/hotspot/share/opto/vectornode.hpp b/src/hotspot/share/opto/vectornode.hpp index 13e6fc2a232aa..efd31766f06a3 100644 --- a/src/hotspot/share/opto/vectornode.hpp +++ b/src/hotspot/share/opto/vectornode.hpp @@ -1474,11 +1474,8 @@ class VectorRearrangeNode : public VectorNode { class VectorLoadShuffleNode : public VectorNode { public: VectorLoadShuffleNode(Node* in, const TypeVect* vt) - : VectorNode(in, vt) { - assert(in->bottom_type()->is_vect()->element_basic_type() == T_BYTE, "must be BYTE"); - } + : VectorNode(in, vt) {} - int GetOutShuffleSize() const { return type2aelembytes(vect_type()->element_basic_type()); } virtual int Opcode() const; }; @@ -1709,14 +1706,11 @@ class VectorBoxAllocateNode : public CallStaticJavaNode { }; class VectorUnboxNode : public VectorNode { - private: - bool _shuffle_to_vector; protected: uint size_of() const { return sizeof(*this); } public: - VectorUnboxNode(Compile* C, const TypeVect* vec_type, Node* obj, Node* mem, bool shuffle_to_vector) + VectorUnboxNode(Compile* C, const TypeVect* vec_type, Node* obj, Node* mem) : VectorNode(mem, obj, vec_type) { - _shuffle_to_vector = shuffle_to_vector; init_class_id(Class_VectorUnbox); init_flags(Flag_is_macro); C->add_macro_node(this); @@ -1727,7 +1721,6 @@ class VectorUnboxNode : public VectorNode { Node* mem() const { return in(1); } virtual Node* Identity(PhaseGVN* phase); Node* Ideal(PhaseGVN* phase, bool can_reshape); - bool is_shuffle_to_vector() { return _shuffle_to_vector; } }; class RotateRightVNode : public VectorNode { diff --git a/src/hotspot/share/prims/vectorSupport.cpp b/src/hotspot/share/prims/vectorSupport.cpp index 4f2e78739ec9c..54fb1cf4d49b8 100644 --- a/src/hotspot/share/prims/vectorSupport.cpp +++ b/src/hotspot/share/prims/vectorSupport.cpp @@ -73,10 +73,6 @@ bool VectorSupport::is_vector_mask(Klass* klass) { return klass->is_subclass_of(vmClasses::vector_VectorMask_klass()); } -bool VectorSupport::is_vector_shuffle(Klass* klass) { - return klass->is_subclass_of(vmClasses::vector_VectorShuffle_klass()); -} - BasicType VectorSupport::klass2bt(InstanceKlass* ik) { assert(ik->is_subclass_of(vmClasses::vector_VectorPayload_klass()), "%s not a VectorPayload", ik->name()->as_C_string()); fieldDescriptor fd; // find_field initializes fd if found @@ -87,9 +83,7 @@ BasicType VectorSupport::klass2bt(InstanceKlass* ik) { assert(fd.is_static(), ""); assert(fd.offset() > 0, ""); - if (is_vector_shuffle(ik)) { - return T_BYTE; - } else if (is_vector_mask(ik)) { + if (is_vector_mask(ik)) { return T_BOOLEAN; } else { // vector and mask oop value = ik->java_mirror()->obj_field(fd.offset()); diff --git a/src/hotspot/share/prims/vectorSupport.hpp b/src/hotspot/share/prims/vectorSupport.hpp index 7302e0060648b..c840a66f7ca11 100644 --- a/src/hotspot/share/prims/vectorSupport.hpp +++ b/src/hotspot/share/prims/vectorSupport.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021, 2022, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2021, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -147,6 +147,5 @@ class VectorSupport : AllStatic { static bool is_vector(Klass* klass); static bool is_vector_mask(Klass* klass); - static bool is_vector_shuffle(Klass* klass); }; #endif // SHARE_PRIMS_VECTORSUPPORT_HPP diff --git a/src/java.base/share/classes/jdk/internal/vm/vector/VectorSupport.java b/src/java.base/share/classes/jdk/internal/vm/vector/VectorSupport.java index a38b72a3b3556..9e04798a89849 100644 --- a/src/java.base/share/classes/jdk/internal/vm/vector/VectorSupport.java +++ b/src/java.base/share/classes/jdk/internal/vm/vector/VectorSupport.java @@ -227,42 +227,6 @@ M indexPartiallyInUpperRange(Class mClass, Class eClass, return defaultImpl.apply(offset, limit); } - /* ============================================================================ */ - public interface ShuffleIotaOperation, - SH extends VectorShuffle> { - SH apply(int length, int start, int step, S s); - } - - @IntrinsicCandidate - public static - , - SH extends VectorShuffle> - SH shuffleIota(Class eClass, Class shClass, S s, - int length, - int start, int step, int wrap, - ShuffleIotaOperation defaultImpl) { - assert isNonCapturingLambda(defaultImpl) : defaultImpl; - return defaultImpl.apply(length, start, step, s); - } - - public interface ShuffleToVectorOperation, - SH extends VectorShuffle> { - V apply(SH sh); - } - - @IntrinsicCandidate - public static - , - SH extends VectorShuffle, - E> - V shuffleToVector(Class> vClass, Class eClass, Class shClass, SH sh, - int length, - ShuffleToVectorOperation defaultImpl) { - assert isNonCapturingLambda(defaultImpl) : defaultImpl; - return defaultImpl.apply(sh); - } - /* ============================================================================ */ public interface IndexOperation, S extends VectorSpecies> { diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/AbstractShuffle.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/AbstractShuffle.java index d648fd86484e0..da20207e16820 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/AbstractShuffle.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/AbstractShuffle.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018, 2022, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2018, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -26,53 +26,16 @@ import java.util.function.IntUnaryOperator; import jdk.internal.vm.annotation.ForceInline; +import jdk.internal.vm.vector.VectorSupport; abstract class AbstractShuffle extends VectorShuffle { static final IntUnaryOperator IDENTITY = i -> i; - // Internal representation allows for a maximum index of 256 + // Internal representation allows for a maximum index of E.MAX_VALUE - 1 // Values are clipped to [-VLENGTH..VLENGTH-1]. - AbstractShuffle(int length, byte[] reorder) { - super(reorder); - assert(length == reorder.length); - assert(indexesInRange(reorder)); - } - - AbstractShuffle(int length, int[] reorder) { - this(length, reorder, 0); - } - - AbstractShuffle(int length, int[] reorder, int offset) { - super(prepare(length, reorder, offset)); - } - - AbstractShuffle(int length, IntUnaryOperator f) { - super(prepare(length, f)); - } - - private static byte[] prepare(int length, int[] reorder, int offset) { - byte[] a = new byte[length]; - for (int i = 0; i < length; i++) { - int si = reorder[offset + i]; - si = partiallyWrapIndex(si, length); - a[i] = (byte) si; - } - return a; - } - - private static byte[] prepare(int length, IntUnaryOperator f) { - byte[] a = new byte[length]; - for (int i = 0; i < a.length; i++) { - int si = f.applyAsInt(i); - si = partiallyWrapIndex(si, length); - a[i] = (byte) si; - } - return a; - } - - byte[] reorder() { - return (byte[])getPayload(); + AbstractShuffle(Object indices) { + super(indices); } /*package-private*/ @@ -84,89 +47,90 @@ public final VectorSpecies vectorSpecies() { return vspecies(); } + /*package-private*/ + abstract AbstractVector toBitsVector(); + + final AbstractVector toBitsVectorTemplate() { + AbstractSpecies dsp = vspecies().asIntegral(); + Class etype = dsp.elementType(); + Class rvtype = dsp.dummyVector().getClass(); + return VectorSupport.convert(VectorSupport.VECTOR_OP_REINTERPRET, + getClass(), etype, length(), + rvtype, etype, length(), + this, dsp, + (v, s) -> v.toBitsVector0()); + } + + abstract AbstractVector toBitsVector0(); + @Override @ForceInline - public void intoArray(int[] a, int offset) { - byte[] reorder = reorder(); - int vlen = reorder.length; - for (int i = 0; i < vlen; i++) { - int sourceIndex = reorder[i]; - assert(sourceIndex >= -vlen && sourceIndex < vlen); - a[offset + i] = sourceIndex; - } + public final Vector toVector() { + return toBitsVector().castShape(vspecies(), 0); } @Override @ForceInline - public int[] toArray() { - byte[] reorder = reorder(); - int[] a = new int[reorder.length]; - intoArray(a, 0); - return a; + public final int[] toArray() { + int[] res = new int[length()]; + intoArray(res, 0); + return res; } - /*package-private*/ + @Override @ForceInline - final - AbstractVector - toVectorTemplate() { - // Note that the values produced by laneSource - // are already clipped. At this point we convert - // them from internal ints (or bytes) into the ETYPE. - // FIXME: Use a conversion intrinsic for this operation. - // https://bugs.openjdk.org/browse/JDK-8225740 - return (AbstractVector) vspecies().fromIntValues(toArray()); + public final VectorShuffle cast(VectorSpecies s) { + if (length() != s.length()) { + throw new IllegalArgumentException("VectorShuffle length and species length differ"); + } + return toBitsVector().toShuffle((AbstractSpecies) s); } + @Override @ForceInline public final VectorShuffle checkIndexes() { if (VectorIntrinsics.VECTOR_ACCESS_OOB_CHECK == 0) { return this; } - Vector shufvec = this.toVector(); - VectorMask vecmask = shufvec.compare(VectorOperators.LT, vspecies().zero()); + Vector shufvec = this.toBitsVector(); + VectorMask vecmask = shufvec.compare(VectorOperators.LT, 0); if (vecmask.anyTrue()) { - byte[] reorder = reorder(); - throw checkIndexFailed(reorder[vecmask.firstTrue()], length()); + int[] indices = toArray(); + throw checkIndexFailed(indices[vecmask.firstTrue()], length()); } return this; } + @Override @ForceInline - public final VectorShuffle wrapIndexes() { - Vector shufvec = this.toVector(); - VectorMask vecmask = shufvec.compare(VectorOperators.LT, vspecies().zero()); - if (vecmask.anyTrue()) { - // FIXME: vectorize this - byte[] reorder = reorder(); - return wrapAndRebuild(reorder); - } - return this; + public final VectorMask laneIsValid() { + Vector shufvec = this.toBitsVector(); + return shufvec.compare(VectorOperators.GE, 0) + .cast(vspecies()); } @ForceInline - public final VectorShuffle wrapAndRebuild(byte[] oldReorder) { - int length = oldReorder.length; - byte[] reorder = new byte[length]; - for (int i = 0; i < length; i++) { - int si = oldReorder[i]; - // FIXME: This does not work unless it's a power of 2. - if ((length & (length - 1)) == 0) { - si += si & length; // power-of-two optimization - } else if (si < 0) { - // non-POT code requires a conditional add - si += length; - } - assert(si >= 0 && si < length); - reorder[i] = (byte) si; - } - return vspecies().dummyVector().shuffleFromBytes(reorder); + @Override + @SuppressWarnings({"rawtypes", "unchecked"}) + public final VectorShuffle rearrange(VectorShuffle shuffle) { + Vector v = toBitsVector(); + return (VectorShuffle) v.rearrange(shuffle.cast(vspecies().asIntegral())) + .toShuffle() + .cast(vspecies()); } @ForceInline - public final VectorMask laneIsValid() { - Vector shufvec = this.toVector(); - return shufvec.compare(VectorOperators.GE, vspecies().zero()); + @Override + @SuppressWarnings({"rawtypes", "unchecked"}) + public final VectorShuffle wrapIndexes() { + Vector v = toBitsVector(); + if ((length() & (length() - 1)) == 0) { + v = v.lanewise(VectorOperators.AND, length() - 1); + } else { + v = v.blend(v.lanewise(VectorOperators.ADD, length()), + v.compare(VectorOperators.LT, 0)); + } + return (VectorShuffle) v.toShuffle().cast(vspecies()); } @Override @@ -220,21 +184,4 @@ private static IndexOutOfBoundsException checkIndexFailed(int index, int laneCou String msg = "required an index in [0.."+max+"] but found "+index; return new IndexOutOfBoundsException(msg); } - - static boolean indexesInRange(byte[] reorder) { - int length = reorder.length; - for (byte si : reorder) { - if (si >= length || si < -length) { - boolean assertsEnabled = false; - assert(assertsEnabled = true); - if (assertsEnabled) { - String msg = ("index "+si+"out of range ["+length+"] in "+ - java.util.Arrays.toString(reorder)); - throw new AssertionError(msg); - } - return false; - } - } - return true; - } } diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/AbstractSpecies.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/AbstractSpecies.java index fac0012e646b4..e6f8a1b8d9661 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/AbstractSpecies.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/AbstractSpecies.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, 2022, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2017, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -48,6 +48,8 @@ abstract class AbstractSpecies extends jdk.internal.vm.vector.VectorSupport.V @Stable final Class> maskType; @Stable + final Class> shuffleType; + @Stable final Function> vectorFactory; @Stable @@ -61,11 +63,13 @@ abstract class AbstractSpecies extends jdk.internal.vm.vector.VectorSupport.V LaneType laneType, Class> vectorType, Class> maskType, + Class> shuffleType, Function> vectorFactory) { this.vectorShape = vectorShape; this.laneType = laneType; this.vectorType = vectorType; this.maskType = maskType; + this.shuffleType = shuffleType; this.vectorFactory = vectorFactory; // derived values: @@ -162,6 +166,11 @@ public final Class> maskType() { return maskType; } + @ForceInline + final Class> shuffleType() { + return shuffleType; + } + @Override @ForceInline public final int elementSize() { diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/AbstractVector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/AbstractVector.java index aac198d6b63d6..8df84cdd89243 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/AbstractVector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/AbstractVector.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019, 2022, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2019, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -188,12 +188,57 @@ ByteVector asByteVectorRawTemplate() { abstract AbstractMask maskFromArray(boolean[] bits); - abstract AbstractShuffle iotaShuffle(); + abstract VectorShuffle toShuffle(AbstractSpecies dsp); - abstract AbstractShuffle iotaShuffle(int start, int step, boolean wrap); + /*package-private*/ + @ForceInline + final VectorShuffle toShuffleTemplate(AbstractSpecies dsp) { + Class etype = vspecies().elementType(); + Class dvtype = dsp.shuffleType(); + Class dtype = dsp.asIntegral().elementType(); + int dlength = dsp.dummyVector().length(); + return VectorSupport.convert(VectorSupport.VECTOR_OP_CAST, + getClass(), etype, length(), + dvtype, dtype, dlength, + this, dsp, + AbstractVector::toShuffle0); + } + + abstract VectorShuffle toShuffle0(AbstractSpecies dsp); + + @ForceInline + public final + VectorShuffle toShuffle() { + return toShuffle(vspecies()); + } + + abstract VectorShuffle iotaShuffle(); - /*do not alias this byte array*/ - abstract AbstractShuffle shuffleFromBytes(byte[] reorder); + @ForceInline + @SuppressWarnings({"rawtypes", "unchecked"}) + final VectorShuffle iotaShuffle(int start, int step, boolean wrap) { + if (start == 0 && step == 1) { + return iotaShuffle(); + } + + if ((length() & (length() - 1)) != 0) { + return wrap ? shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i * step + start, length()))) + : shuffleFromOp(i -> i * step + start); + } + + AbstractSpecies species = vspecies().asIntegral(); + Vector iota = species.iota(); + iota = iota.lanewise(VectorOperators.MUL, step) + .lanewise(VectorOperators.ADD, start); + Vector wrapped = iota.lanewise(VectorOperators.AND, length() - 1); + + if (!wrap) { + Vector wrappedEx = wrapped.lanewise(VectorOperators.SUB, length()); + VectorMask mask = wrapped.compare(VectorOperators.EQ, iota); + wrapped = wrappedEx.blend(wrapped, mask); + } + return ((AbstractVector) wrapped).toShuffle(vspecies()); + } abstract AbstractShuffle shuffleFromArray(int[] indexes, int i); diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte128Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte128Vector.java index 12a6c80f21e5b..fdf1f55dc69b6 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte128Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte128Vector.java @@ -141,24 +141,9 @@ Byte128Mask maskFromArray(boolean[] bits) { @ForceInline Byte128Shuffle iotaShuffle() { return Byte128Shuffle.IOTA; } - @ForceInline - Byte128Shuffle iotaShuffle(int start, int step, boolean wrap) { - if (wrap) { - return (Byte128Shuffle)VectorSupport.shuffleIota(ETYPE, Byte128Shuffle.class, VSPECIES, VLENGTH, start, step, 1, - (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i*lstep + lstart, l)))); - } else { - return (Byte128Shuffle)VectorSupport.shuffleIota(ETYPE, Byte128Shuffle.class, VSPECIES, VLENGTH, start, step, 0, - (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (i*lstep + lstart))); - } - } - @Override @ForceInline - Byte128Shuffle shuffleFromBytes(byte[] reorder) { return new Byte128Shuffle(reorder); } - - @Override - @ForceInline - Byte128Shuffle shuffleFromArray(int[] indexes, int i) { return new Byte128Shuffle(indexes, i); } + Byte128Shuffle shuffleFromArray(int[] indices, int i) { return new Byte128Shuffle(indices, i); } @Override @ForceInline @@ -357,9 +342,11 @@ public final long reduceLanesToLong(VectorOperators.Associative op, return (long) super.reduceLanesTemplate(op, Byte128Mask.class, (Byte128Mask) m); // specialized } + @Override @ForceInline - public VectorShuffle toShuffle() { - return super.toShuffleTemplate(Byte128Shuffle.class); // specialize + public final + VectorShuffle toShuffle(AbstractSpecies dsp) { + return super.toShuffleTemplate(dsp); } // Specialized unary testing @@ -812,23 +799,26 @@ static final class Byte128Shuffle extends AbstractShuffle { static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM static final Class ETYPE = byte.class; // used by the JVM - Byte128Shuffle(byte[] reorder) { - super(VLENGTH, reorder); + Byte128Shuffle(byte[] indices) { + super(indices); + assert(VLENGTH == indices.length); + assert(indicesInRange(indices)); } - public Byte128Shuffle(int[] reorder) { - super(VLENGTH, reorder); + Byte128Shuffle(int[] indices, int i) { + this(prepare(indices, i)); } - public Byte128Shuffle(int[] reorder, int i) { - super(VLENGTH, reorder, i); + Byte128Shuffle(IntUnaryOperator fn) { + this(prepare(fn)); } - public Byte128Shuffle(IntUnaryOperator fn) { - super(VLENGTH, fn); + byte[] indices() { + return (byte[])getPayload(); } @Override + @ForceInline public ByteSpecies vspecies() { return VSPECIES; } @@ -843,33 +833,76 @@ public ByteSpecies vspecies() { @Override @ForceInline - public Byte128Vector toVector() { - return VectorSupport.shuffleToVector(VCLASS, ETYPE, Byte128Shuffle.class, this, VLENGTH, - (s) -> ((Byte128Vector)(((AbstractShuffle)(s)).toVectorTemplate()))); + Byte128Vector toBitsVector() { + return (Byte128Vector) super.toBitsVectorTemplate(); } @Override @ForceInline - public VectorShuffle cast(VectorSpecies s) { - AbstractSpecies species = (AbstractSpecies) s; - if (length() != species.laneCount()) - throw new IllegalArgumentException("VectorShuffle length and species length differ"); - int[] shuffleArray = toArray(); - return s.shuffleFromArray(shuffleArray, 0).check(s); + ByteVector toBitsVector0() { + return Byte128Vector.VSPECIES.dummyVector().vectorFactory(indices()); } + @Override @ForceInline + public int laneSource(int i) { + return (int)toBitsVector().lane(i); + } + @Override - public Byte128Shuffle rearrange(VectorShuffle shuffle) { - Byte128Shuffle s = (Byte128Shuffle) shuffle; - byte[] reorder1 = reorder(); - byte[] reorder2 = s.reorder(); - byte[] r = new byte[reorder1.length]; - for (int i = 0; i < reorder1.length; i++) { - int ssi = reorder2[i]; - r[i] = reorder1[ssi]; // throws on exceptional index + @ForceInline + public void intoArray(int[] a, int offset) { + VectorSpecies species = IntVector.SPECIES_128; + Vector v = toBitsVector(); + v.convertShape(VectorOperators.B2I, species, 0) + .reinterpretAsInts() + .intoArray(a, offset); + v.convertShape(VectorOperators.B2I, species, 1) + .reinterpretAsInts() + .intoArray(a, offset + species.length()); + v.convertShape(VectorOperators.B2I, species, 2) + .reinterpretAsInts() + .intoArray(a, offset + species.length() * 2); + v.convertShape(VectorOperators.B2I, species, 3) + .reinterpretAsInts() + .intoArray(a, offset + species.length() * 3); + } + + private static byte[] prepare(int[] indices, int offset) { + byte[] a = new byte[VLENGTH]; + for (int i = 0; i < VLENGTH; i++) { + int si = indices[offset + i]; + si = partiallyWrapIndex(si, VLENGTH); + a[i] = (byte)si; + } + return a; + } + + private static byte[] prepare(IntUnaryOperator f) { + byte[] a = new byte[VLENGTH]; + for (int i = 0; i < VLENGTH; i++) { + int si = f.applyAsInt(i); + si = partiallyWrapIndex(si, VLENGTH); + a[i] = (byte)si; + } + return a; + } + + private static boolean indicesInRange(byte[] indices) { + int length = indices.length; + for (byte si : indices) { + if (si >= (byte)length || si < (byte)(-length)) { + boolean assertsEnabled = false; + assert(assertsEnabled = true); + if (assertsEnabled) { + String msg = ("index "+si+"out of range ["+length+"] in "+ + java.util.Arrays.toString(indices)); + throw new AssertionError(msg); + } + return false; + } } - return new Byte128Shuffle(r); + return true; } } diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte256Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte256Vector.java index e205071680378..c9496d36b1392 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte256Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte256Vector.java @@ -141,24 +141,9 @@ Byte256Mask maskFromArray(boolean[] bits) { @ForceInline Byte256Shuffle iotaShuffle() { return Byte256Shuffle.IOTA; } - @ForceInline - Byte256Shuffle iotaShuffle(int start, int step, boolean wrap) { - if (wrap) { - return (Byte256Shuffle)VectorSupport.shuffleIota(ETYPE, Byte256Shuffle.class, VSPECIES, VLENGTH, start, step, 1, - (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i*lstep + lstart, l)))); - } else { - return (Byte256Shuffle)VectorSupport.shuffleIota(ETYPE, Byte256Shuffle.class, VSPECIES, VLENGTH, start, step, 0, - (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (i*lstep + lstart))); - } - } - - @Override - @ForceInline - Byte256Shuffle shuffleFromBytes(byte[] reorder) { return new Byte256Shuffle(reorder); } - @Override @ForceInline - Byte256Shuffle shuffleFromArray(int[] indexes, int i) { return new Byte256Shuffle(indexes, i); } + Byte256Shuffle shuffleFromArray(int[] indices, int i) { return new Byte256Shuffle(indices, i); } @Override @ForceInline @@ -357,9 +342,11 @@ public final long reduceLanesToLong(VectorOperators.Associative op, return (long) super.reduceLanesTemplate(op, Byte256Mask.class, (Byte256Mask) m); // specialized } + @Override @ForceInline - public VectorShuffle toShuffle() { - return super.toShuffleTemplate(Byte256Shuffle.class); // specialize + public final + VectorShuffle toShuffle(AbstractSpecies dsp) { + return super.toShuffleTemplate(dsp); } // Specialized unary testing @@ -844,23 +831,26 @@ static final class Byte256Shuffle extends AbstractShuffle { static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM static final Class ETYPE = byte.class; // used by the JVM - Byte256Shuffle(byte[] reorder) { - super(VLENGTH, reorder); + Byte256Shuffle(byte[] indices) { + super(indices); + assert(VLENGTH == indices.length); + assert(indicesInRange(indices)); } - public Byte256Shuffle(int[] reorder) { - super(VLENGTH, reorder); + Byte256Shuffle(int[] indices, int i) { + this(prepare(indices, i)); } - public Byte256Shuffle(int[] reorder, int i) { - super(VLENGTH, reorder, i); + Byte256Shuffle(IntUnaryOperator fn) { + this(prepare(fn)); } - public Byte256Shuffle(IntUnaryOperator fn) { - super(VLENGTH, fn); + byte[] indices() { + return (byte[])getPayload(); } @Override + @ForceInline public ByteSpecies vspecies() { return VSPECIES; } @@ -875,33 +865,76 @@ public ByteSpecies vspecies() { @Override @ForceInline - public Byte256Vector toVector() { - return VectorSupport.shuffleToVector(VCLASS, ETYPE, Byte256Shuffle.class, this, VLENGTH, - (s) -> ((Byte256Vector)(((AbstractShuffle)(s)).toVectorTemplate()))); + Byte256Vector toBitsVector() { + return (Byte256Vector) super.toBitsVectorTemplate(); } @Override @ForceInline - public VectorShuffle cast(VectorSpecies s) { - AbstractSpecies species = (AbstractSpecies) s; - if (length() != species.laneCount()) - throw new IllegalArgumentException("VectorShuffle length and species length differ"); - int[] shuffleArray = toArray(); - return s.shuffleFromArray(shuffleArray, 0).check(s); + ByteVector toBitsVector0() { + return Byte256Vector.VSPECIES.dummyVector().vectorFactory(indices()); } + @Override @ForceInline + public int laneSource(int i) { + return (int)toBitsVector().lane(i); + } + @Override - public Byte256Shuffle rearrange(VectorShuffle shuffle) { - Byte256Shuffle s = (Byte256Shuffle) shuffle; - byte[] reorder1 = reorder(); - byte[] reorder2 = s.reorder(); - byte[] r = new byte[reorder1.length]; - for (int i = 0; i < reorder1.length; i++) { - int ssi = reorder2[i]; - r[i] = reorder1[ssi]; // throws on exceptional index + @ForceInline + public void intoArray(int[] a, int offset) { + VectorSpecies species = IntVector.SPECIES_256; + Vector v = toBitsVector(); + v.convertShape(VectorOperators.B2I, species, 0) + .reinterpretAsInts() + .intoArray(a, offset); + v.convertShape(VectorOperators.B2I, species, 1) + .reinterpretAsInts() + .intoArray(a, offset + species.length()); + v.convertShape(VectorOperators.B2I, species, 2) + .reinterpretAsInts() + .intoArray(a, offset + species.length() * 2); + v.convertShape(VectorOperators.B2I, species, 3) + .reinterpretAsInts() + .intoArray(a, offset + species.length() * 3); + } + + private static byte[] prepare(int[] indices, int offset) { + byte[] a = new byte[VLENGTH]; + for (int i = 0; i < VLENGTH; i++) { + int si = indices[offset + i]; + si = partiallyWrapIndex(si, VLENGTH); + a[i] = (byte)si; + } + return a; + } + + private static byte[] prepare(IntUnaryOperator f) { + byte[] a = new byte[VLENGTH]; + for (int i = 0; i < VLENGTH; i++) { + int si = f.applyAsInt(i); + si = partiallyWrapIndex(si, VLENGTH); + a[i] = (byte)si; + } + return a; + } + + private static boolean indicesInRange(byte[] indices) { + int length = indices.length; + for (byte si : indices) { + if (si >= (byte)length || si < (byte)(-length)) { + boolean assertsEnabled = false; + assert(assertsEnabled = true); + if (assertsEnabled) { + String msg = ("index "+si+"out of range ["+length+"] in "+ + java.util.Arrays.toString(indices)); + throw new AssertionError(msg); + } + return false; + } } - return new Byte256Shuffle(r); + return true; } } diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte512Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte512Vector.java index a8a5875ea7b71..10cbb3386c74e 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte512Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte512Vector.java @@ -141,24 +141,9 @@ Byte512Mask maskFromArray(boolean[] bits) { @ForceInline Byte512Shuffle iotaShuffle() { return Byte512Shuffle.IOTA; } - @ForceInline - Byte512Shuffle iotaShuffle(int start, int step, boolean wrap) { - if (wrap) { - return (Byte512Shuffle)VectorSupport.shuffleIota(ETYPE, Byte512Shuffle.class, VSPECIES, VLENGTH, start, step, 1, - (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i*lstep + lstart, l)))); - } else { - return (Byte512Shuffle)VectorSupport.shuffleIota(ETYPE, Byte512Shuffle.class, VSPECIES, VLENGTH, start, step, 0, - (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (i*lstep + lstart))); - } - } - - @Override - @ForceInline - Byte512Shuffle shuffleFromBytes(byte[] reorder) { return new Byte512Shuffle(reorder); } - @Override @ForceInline - Byte512Shuffle shuffleFromArray(int[] indexes, int i) { return new Byte512Shuffle(indexes, i); } + Byte512Shuffle shuffleFromArray(int[] indices, int i) { return new Byte512Shuffle(indices, i); } @Override @ForceInline @@ -357,9 +342,11 @@ public final long reduceLanesToLong(VectorOperators.Associative op, return (long) super.reduceLanesTemplate(op, Byte512Mask.class, (Byte512Mask) m); // specialized } + @Override @ForceInline - public VectorShuffle toShuffle() { - return super.toShuffleTemplate(Byte512Shuffle.class); // specialize + public final + VectorShuffle toShuffle(AbstractSpecies dsp) { + return super.toShuffleTemplate(dsp); } // Specialized unary testing @@ -908,23 +895,26 @@ static final class Byte512Shuffle extends AbstractShuffle { static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM static final Class ETYPE = byte.class; // used by the JVM - Byte512Shuffle(byte[] reorder) { - super(VLENGTH, reorder); + Byte512Shuffle(byte[] indices) { + super(indices); + assert(VLENGTH == indices.length); + assert(indicesInRange(indices)); } - public Byte512Shuffle(int[] reorder) { - super(VLENGTH, reorder); + Byte512Shuffle(int[] indices, int i) { + this(prepare(indices, i)); } - public Byte512Shuffle(int[] reorder, int i) { - super(VLENGTH, reorder, i); + Byte512Shuffle(IntUnaryOperator fn) { + this(prepare(fn)); } - public Byte512Shuffle(IntUnaryOperator fn) { - super(VLENGTH, fn); + byte[] indices() { + return (byte[])getPayload(); } @Override + @ForceInline public ByteSpecies vspecies() { return VSPECIES; } @@ -939,33 +929,76 @@ public ByteSpecies vspecies() { @Override @ForceInline - public Byte512Vector toVector() { - return VectorSupport.shuffleToVector(VCLASS, ETYPE, Byte512Shuffle.class, this, VLENGTH, - (s) -> ((Byte512Vector)(((AbstractShuffle)(s)).toVectorTemplate()))); + Byte512Vector toBitsVector() { + return (Byte512Vector) super.toBitsVectorTemplate(); } @Override @ForceInline - public VectorShuffle cast(VectorSpecies s) { - AbstractSpecies species = (AbstractSpecies) s; - if (length() != species.laneCount()) - throw new IllegalArgumentException("VectorShuffle length and species length differ"); - int[] shuffleArray = toArray(); - return s.shuffleFromArray(shuffleArray, 0).check(s); + ByteVector toBitsVector0() { + return Byte512Vector.VSPECIES.dummyVector().vectorFactory(indices()); } + @Override @ForceInline + public int laneSource(int i) { + return (int)toBitsVector().lane(i); + } + @Override - public Byte512Shuffle rearrange(VectorShuffle shuffle) { - Byte512Shuffle s = (Byte512Shuffle) shuffle; - byte[] reorder1 = reorder(); - byte[] reorder2 = s.reorder(); - byte[] r = new byte[reorder1.length]; - for (int i = 0; i < reorder1.length; i++) { - int ssi = reorder2[i]; - r[i] = reorder1[ssi]; // throws on exceptional index + @ForceInline + public void intoArray(int[] a, int offset) { + VectorSpecies species = IntVector.SPECIES_512; + Vector v = toBitsVector(); + v.convertShape(VectorOperators.B2I, species, 0) + .reinterpretAsInts() + .intoArray(a, offset); + v.convertShape(VectorOperators.B2I, species, 1) + .reinterpretAsInts() + .intoArray(a, offset + species.length()); + v.convertShape(VectorOperators.B2I, species, 2) + .reinterpretAsInts() + .intoArray(a, offset + species.length() * 2); + v.convertShape(VectorOperators.B2I, species, 3) + .reinterpretAsInts() + .intoArray(a, offset + species.length() * 3); + } + + private static byte[] prepare(int[] indices, int offset) { + byte[] a = new byte[VLENGTH]; + for (int i = 0; i < VLENGTH; i++) { + int si = indices[offset + i]; + si = partiallyWrapIndex(si, VLENGTH); + a[i] = (byte)si; + } + return a; + } + + private static byte[] prepare(IntUnaryOperator f) { + byte[] a = new byte[VLENGTH]; + for (int i = 0; i < VLENGTH; i++) { + int si = f.applyAsInt(i); + si = partiallyWrapIndex(si, VLENGTH); + a[i] = (byte)si; + } + return a; + } + + private static boolean indicesInRange(byte[] indices) { + int length = indices.length; + for (byte si : indices) { + if (si >= (byte)length || si < (byte)(-length)) { + boolean assertsEnabled = false; + assert(assertsEnabled = true); + if (assertsEnabled) { + String msg = ("index "+si+"out of range ["+length+"] in "+ + java.util.Arrays.toString(indices)); + throw new AssertionError(msg); + } + return false; + } } - return new Byte512Shuffle(r); + return true; } } diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte64Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte64Vector.java index db3e925294495..6ddd22c3627b2 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte64Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte64Vector.java @@ -141,24 +141,9 @@ Byte64Mask maskFromArray(boolean[] bits) { @ForceInline Byte64Shuffle iotaShuffle() { return Byte64Shuffle.IOTA; } - @ForceInline - Byte64Shuffle iotaShuffle(int start, int step, boolean wrap) { - if (wrap) { - return (Byte64Shuffle)VectorSupport.shuffleIota(ETYPE, Byte64Shuffle.class, VSPECIES, VLENGTH, start, step, 1, - (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i*lstep + lstart, l)))); - } else { - return (Byte64Shuffle)VectorSupport.shuffleIota(ETYPE, Byte64Shuffle.class, VSPECIES, VLENGTH, start, step, 0, - (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (i*lstep + lstart))); - } - } - @Override @ForceInline - Byte64Shuffle shuffleFromBytes(byte[] reorder) { return new Byte64Shuffle(reorder); } - - @Override - @ForceInline - Byte64Shuffle shuffleFromArray(int[] indexes, int i) { return new Byte64Shuffle(indexes, i); } + Byte64Shuffle shuffleFromArray(int[] indices, int i) { return new Byte64Shuffle(indices, i); } @Override @ForceInline @@ -357,9 +342,11 @@ public final long reduceLanesToLong(VectorOperators.Associative op, return (long) super.reduceLanesTemplate(op, Byte64Mask.class, (Byte64Mask) m); // specialized } + @Override @ForceInline - public VectorShuffle toShuffle() { - return super.toShuffleTemplate(Byte64Shuffle.class); // specialize + public final + VectorShuffle toShuffle(AbstractSpecies dsp) { + return super.toShuffleTemplate(dsp); } // Specialized unary testing @@ -796,23 +783,26 @@ static final class Byte64Shuffle extends AbstractShuffle { static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM static final Class ETYPE = byte.class; // used by the JVM - Byte64Shuffle(byte[] reorder) { - super(VLENGTH, reorder); + Byte64Shuffle(byte[] indices) { + super(indices); + assert(VLENGTH == indices.length); + assert(indicesInRange(indices)); } - public Byte64Shuffle(int[] reorder) { - super(VLENGTH, reorder); + Byte64Shuffle(int[] indices, int i) { + this(prepare(indices, i)); } - public Byte64Shuffle(int[] reorder, int i) { - super(VLENGTH, reorder, i); + Byte64Shuffle(IntUnaryOperator fn) { + this(prepare(fn)); } - public Byte64Shuffle(IntUnaryOperator fn) { - super(VLENGTH, fn); + byte[] indices() { + return (byte[])getPayload(); } @Override + @ForceInline public ByteSpecies vspecies() { return VSPECIES; } @@ -827,33 +817,76 @@ public ByteSpecies vspecies() { @Override @ForceInline - public Byte64Vector toVector() { - return VectorSupport.shuffleToVector(VCLASS, ETYPE, Byte64Shuffle.class, this, VLENGTH, - (s) -> ((Byte64Vector)(((AbstractShuffle)(s)).toVectorTemplate()))); + Byte64Vector toBitsVector() { + return (Byte64Vector) super.toBitsVectorTemplate(); } @Override @ForceInline - public VectorShuffle cast(VectorSpecies s) { - AbstractSpecies species = (AbstractSpecies) s; - if (length() != species.laneCount()) - throw new IllegalArgumentException("VectorShuffle length and species length differ"); - int[] shuffleArray = toArray(); - return s.shuffleFromArray(shuffleArray, 0).check(s); + ByteVector toBitsVector0() { + return Byte64Vector.VSPECIES.dummyVector().vectorFactory(indices()); } + @Override @ForceInline + public int laneSource(int i) { + return (int)toBitsVector().lane(i); + } + @Override - public Byte64Shuffle rearrange(VectorShuffle shuffle) { - Byte64Shuffle s = (Byte64Shuffle) shuffle; - byte[] reorder1 = reorder(); - byte[] reorder2 = s.reorder(); - byte[] r = new byte[reorder1.length]; - for (int i = 0; i < reorder1.length; i++) { - int ssi = reorder2[i]; - r[i] = reorder1[ssi]; // throws on exceptional index + @ForceInline + public void intoArray(int[] a, int offset) { + VectorSpecies species = IntVector.SPECIES_64; + Vector v = toBitsVector(); + v.convertShape(VectorOperators.B2I, species, 0) + .reinterpretAsInts() + .intoArray(a, offset); + v.convertShape(VectorOperators.B2I, species, 1) + .reinterpretAsInts() + .intoArray(a, offset + species.length()); + v.convertShape(VectorOperators.B2I, species, 2) + .reinterpretAsInts() + .intoArray(a, offset + species.length() * 2); + v.convertShape(VectorOperators.B2I, species, 3) + .reinterpretAsInts() + .intoArray(a, offset + species.length() * 3); + } + + private static byte[] prepare(int[] indices, int offset) { + byte[] a = new byte[VLENGTH]; + for (int i = 0; i < VLENGTH; i++) { + int si = indices[offset + i]; + si = partiallyWrapIndex(si, VLENGTH); + a[i] = (byte)si; + } + return a; + } + + private static byte[] prepare(IntUnaryOperator f) { + byte[] a = new byte[VLENGTH]; + for (int i = 0; i < VLENGTH; i++) { + int si = f.applyAsInt(i); + si = partiallyWrapIndex(si, VLENGTH); + a[i] = (byte)si; + } + return a; + } + + private static boolean indicesInRange(byte[] indices) { + int length = indices.length; + for (byte si : indices) { + if (si >= (byte)length || si < (byte)(-length)) { + boolean assertsEnabled = false; + assert(assertsEnabled = true); + if (assertsEnabled) { + String msg = ("index "+si+"out of range ["+length+"] in "+ + java.util.Arrays.toString(indices)); + throw new AssertionError(msg); + } + return false; + } } - return new Byte64Shuffle(r); + return true; } } diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ByteMaxVector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ByteMaxVector.java index 0afdeab13dc40..83404b1634a09 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ByteMaxVector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ByteMaxVector.java @@ -141,24 +141,9 @@ ByteMaxMask maskFromArray(boolean[] bits) { @ForceInline ByteMaxShuffle iotaShuffle() { return ByteMaxShuffle.IOTA; } - @ForceInline - ByteMaxShuffle iotaShuffle(int start, int step, boolean wrap) { - if (wrap) { - return (ByteMaxShuffle)VectorSupport.shuffleIota(ETYPE, ByteMaxShuffle.class, VSPECIES, VLENGTH, start, step, 1, - (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i*lstep + lstart, l)))); - } else { - return (ByteMaxShuffle)VectorSupport.shuffleIota(ETYPE, ByteMaxShuffle.class, VSPECIES, VLENGTH, start, step, 0, - (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (i*lstep + lstart))); - } - } - @Override @ForceInline - ByteMaxShuffle shuffleFromBytes(byte[] reorder) { return new ByteMaxShuffle(reorder); } - - @Override - @ForceInline - ByteMaxShuffle shuffleFromArray(int[] indexes, int i) { return new ByteMaxShuffle(indexes, i); } + ByteMaxShuffle shuffleFromArray(int[] indices, int i) { return new ByteMaxShuffle(indices, i); } @Override @ForceInline @@ -357,9 +342,11 @@ public final long reduceLanesToLong(VectorOperators.Associative op, return (long) super.reduceLanesTemplate(op, ByteMaxMask.class, (ByteMaxMask) m); // specialized } + @Override @ForceInline - public VectorShuffle toShuffle() { - return super.toShuffleTemplate(ByteMaxShuffle.class); // specialize + public final + VectorShuffle toShuffle(AbstractSpecies dsp) { + return super.toShuffleTemplate(dsp); } // Specialized unary testing @@ -782,23 +769,26 @@ static final class ByteMaxShuffle extends AbstractShuffle { static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM static final Class ETYPE = byte.class; // used by the JVM - ByteMaxShuffle(byte[] reorder) { - super(VLENGTH, reorder); + ByteMaxShuffle(byte[] indices) { + super(indices); + assert(VLENGTH == indices.length); + assert(indicesInRange(indices)); } - public ByteMaxShuffle(int[] reorder) { - super(VLENGTH, reorder); + ByteMaxShuffle(int[] indices, int i) { + this(prepare(indices, i)); } - public ByteMaxShuffle(int[] reorder, int i) { - super(VLENGTH, reorder, i); + ByteMaxShuffle(IntUnaryOperator fn) { + this(prepare(fn)); } - public ByteMaxShuffle(IntUnaryOperator fn) { - super(VLENGTH, fn); + byte[] indices() { + return (byte[])getPayload(); } @Override + @ForceInline public ByteSpecies vspecies() { return VSPECIES; } @@ -813,33 +803,76 @@ public ByteSpecies vspecies() { @Override @ForceInline - public ByteMaxVector toVector() { - return VectorSupport.shuffleToVector(VCLASS, ETYPE, ByteMaxShuffle.class, this, VLENGTH, - (s) -> ((ByteMaxVector)(((AbstractShuffle)(s)).toVectorTemplate()))); + ByteMaxVector toBitsVector() { + return (ByteMaxVector) super.toBitsVectorTemplate(); } @Override @ForceInline - public VectorShuffle cast(VectorSpecies s) { - AbstractSpecies species = (AbstractSpecies) s; - if (length() != species.laneCount()) - throw new IllegalArgumentException("VectorShuffle length and species length differ"); - int[] shuffleArray = toArray(); - return s.shuffleFromArray(shuffleArray, 0).check(s); + ByteVector toBitsVector0() { + return ByteMaxVector.VSPECIES.dummyVector().vectorFactory(indices()); } + @Override @ForceInline + public int laneSource(int i) { + return (int)toBitsVector().lane(i); + } + @Override - public ByteMaxShuffle rearrange(VectorShuffle shuffle) { - ByteMaxShuffle s = (ByteMaxShuffle) shuffle; - byte[] reorder1 = reorder(); - byte[] reorder2 = s.reorder(); - byte[] r = new byte[reorder1.length]; - for (int i = 0; i < reorder1.length; i++) { - int ssi = reorder2[i]; - r[i] = reorder1[ssi]; // throws on exceptional index + @ForceInline + public void intoArray(int[] a, int offset) { + VectorSpecies species = IntVector.SPECIES_MAX; + Vector v = toBitsVector(); + v.convertShape(VectorOperators.B2I, species, 0) + .reinterpretAsInts() + .intoArray(a, offset); + v.convertShape(VectorOperators.B2I, species, 1) + .reinterpretAsInts() + .intoArray(a, offset + species.length()); + v.convertShape(VectorOperators.B2I, species, 2) + .reinterpretAsInts() + .intoArray(a, offset + species.length() * 2); + v.convertShape(VectorOperators.B2I, species, 3) + .reinterpretAsInts() + .intoArray(a, offset + species.length() * 3); + } + + private static byte[] prepare(int[] indices, int offset) { + byte[] a = new byte[VLENGTH]; + for (int i = 0; i < VLENGTH; i++) { + int si = indices[offset + i]; + si = partiallyWrapIndex(si, VLENGTH); + a[i] = (byte)si; + } + return a; + } + + private static byte[] prepare(IntUnaryOperator f) { + byte[] a = new byte[VLENGTH]; + for (int i = 0; i < VLENGTH; i++) { + int si = f.applyAsInt(i); + si = partiallyWrapIndex(si, VLENGTH); + a[i] = (byte)si; + } + return a; + } + + private static boolean indicesInRange(byte[] indices) { + int length = indices.length; + for (byte si : indices) { + if (si >= (byte)length || si < (byte)(-length)) { + boolean assertsEnabled = false; + assert(assertsEnabled = true); + if (assertsEnabled) { + String msg = ("index "+si+"out of range ["+length+"] in "+ + java.util.Arrays.toString(indices)); + throw new AssertionError(msg); + } + return false; + } } - return new ByteMaxShuffle(r); + return true; } } diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ByteVector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ByteVector.java index 0ae566fda7f2f..87a70e8c28ea6 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ByteVector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ByteVector.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, 2022, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2017, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -1071,7 +1071,7 @@ private static VectorBroadcastIntOp> broadcastIntOp // and broadcast, but it would be more surprising not to continue // the obvious pattern started by unary and binary. - /** + /** * {@inheritDoc} * @see #lanewise(VectorOperators.Ternary,byte,byte,VectorMask) * @see #lanewise(VectorOperators.Ternary,Vector,byte,VectorMask) @@ -2480,8 +2480,8 @@ ByteVector rearrangeTemplate(Class shuffletype, } @ForceInline - private final - VectorShuffle toShuffle0(ByteSpecies dsp) { + final + VectorShuffle toShuffle0(AbstractSpecies dsp) { byte[] a = toArray(); int[] sa = new int[a.length]; for (int i = 0; i < a.length; i++) { @@ -2490,18 +2490,6 @@ VectorShuffle toShuffle0(ByteSpecies dsp) { return VectorShuffle.fromArray(dsp, sa, 0); } - /*package-private*/ - @ForceInline - final - VectorShuffle toShuffleTemplate(Class shuffleType) { - ByteSpecies vsp = vspecies(); - return VectorSupport.convert(VectorSupport.VECTOR_OP_CAST, - getClass(), byte.class, length(), - shuffleType, byte.class, length(), - this, vsp, - ByteVector::toShuffle0); - } - /** * {@inheritDoc} * @since 19 @@ -4096,9 +4084,10 @@ static final class ByteSpecies extends AbstractSpecies { private ByteSpecies(VectorShape shape, Class vectorType, Class> maskType, + Class> shuffleType, Function vectorFactory) { super(shape, LaneType.of(byte.class), - vectorType, maskType, + vectorType, maskType, shuffleType, vectorFactory); assert(this.elementSize() == Byte.SIZE); } @@ -4374,6 +4363,7 @@ static ByteSpecies species(VectorShape s) { = new ByteSpecies(VectorShape.S_64_BIT, Byte64Vector.class, Byte64Vector.Byte64Mask.class, + Byte64Vector.Byte64Shuffle.class, Byte64Vector::new); /** Species representing {@link ByteVector}s of {@link VectorShape#S_128_BIT VectorShape.S_128_BIT}. */ @@ -4381,6 +4371,7 @@ static ByteSpecies species(VectorShape s) { = new ByteSpecies(VectorShape.S_128_BIT, Byte128Vector.class, Byte128Vector.Byte128Mask.class, + Byte128Vector.Byte128Shuffle.class, Byte128Vector::new); /** Species representing {@link ByteVector}s of {@link VectorShape#S_256_BIT VectorShape.S_256_BIT}. */ @@ -4388,6 +4379,7 @@ static ByteSpecies species(VectorShape s) { = new ByteSpecies(VectorShape.S_256_BIT, Byte256Vector.class, Byte256Vector.Byte256Mask.class, + Byte256Vector.Byte256Shuffle.class, Byte256Vector::new); /** Species representing {@link ByteVector}s of {@link VectorShape#S_512_BIT VectorShape.S_512_BIT}. */ @@ -4395,6 +4387,7 @@ static ByteSpecies species(VectorShape s) { = new ByteSpecies(VectorShape.S_512_BIT, Byte512Vector.class, Byte512Vector.Byte512Mask.class, + Byte512Vector.Byte512Shuffle.class, Byte512Vector::new); /** Species representing {@link ByteVector}s of {@link VectorShape#S_Max_BIT VectorShape.S_Max_BIT}. */ @@ -4402,6 +4395,7 @@ static ByteSpecies species(VectorShape s) { = new ByteSpecies(VectorShape.S_Max_BIT, ByteMaxVector.class, ByteMaxVector.ByteMaxMask.class, + ByteMaxVector.ByteMaxShuffle.class, ByteMaxVector::new); /** diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double128Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double128Vector.java index 2c5706fe2f059..d747cb5136d2e 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double128Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double128Vector.java @@ -141,24 +141,9 @@ Double128Mask maskFromArray(boolean[] bits) { @ForceInline Double128Shuffle iotaShuffle() { return Double128Shuffle.IOTA; } - @ForceInline - Double128Shuffle iotaShuffle(int start, int step, boolean wrap) { - if (wrap) { - return (Double128Shuffle)VectorSupport.shuffleIota(ETYPE, Double128Shuffle.class, VSPECIES, VLENGTH, start, step, 1, - (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i*lstep + lstart, l)))); - } else { - return (Double128Shuffle)VectorSupport.shuffleIota(ETYPE, Double128Shuffle.class, VSPECIES, VLENGTH, start, step, 0, - (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (i*lstep + lstart))); - } - } - - @Override - @ForceInline - Double128Shuffle shuffleFromBytes(byte[] reorder) { return new Double128Shuffle(reorder); } - @Override @ForceInline - Double128Shuffle shuffleFromArray(int[] indexes, int i) { return new Double128Shuffle(indexes, i); } + Double128Shuffle shuffleFromArray(int[] indices, int i) { return new Double128Shuffle(indices, i); } @Override @ForceInline @@ -344,9 +329,11 @@ public final long reduceLanesToLong(VectorOperators.Associative op, return (long) super.reduceLanesTemplate(op, Double128Mask.class, (Double128Mask) m); // specialized } + @Override @ForceInline - public VectorShuffle toShuffle() { - return super.toShuffleTemplate(Double128Shuffle.class); // specialize + public final + VectorShuffle toShuffle(AbstractSpecies dsp) { + return super.toShuffleTemplate(dsp); } // Specialized unary testing @@ -771,25 +758,28 @@ static Double128Mask maskAll(boolean bit) { static final class Double128Shuffle extends AbstractShuffle { static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM - static final Class ETYPE = double.class; // used by the JVM + static final Class ETYPE = long.class; // used by the JVM - Double128Shuffle(byte[] reorder) { - super(VLENGTH, reorder); + Double128Shuffle(long[] indices) { + super(indices); + assert(VLENGTH == indices.length); + assert(indicesInRange(indices)); } - public Double128Shuffle(int[] reorder) { - super(VLENGTH, reorder); + Double128Shuffle(int[] indices, int i) { + this(prepare(indices, i)); } - public Double128Shuffle(int[] reorder, int i) { - super(VLENGTH, reorder, i); + Double128Shuffle(IntUnaryOperator fn) { + this(prepare(fn)); } - public Double128Shuffle(IntUnaryOperator fn) { - super(VLENGTH, fn); + long[] indices() { + return (long[])getPayload(); } @Override + @ForceInline public DoubleSpecies vspecies() { return VSPECIES; } @@ -797,40 +787,76 @@ public DoubleSpecies vspecies() { static { // There must be enough bits in the shuffle lanes to encode // VLENGTH valid indexes and VLENGTH exceptional ones. - assert(VLENGTH < Byte.MAX_VALUE); - assert(Byte.MIN_VALUE <= -VLENGTH); + assert(VLENGTH < Long.MAX_VALUE); + assert(Long.MIN_VALUE <= -VLENGTH); } static final Double128Shuffle IOTA = new Double128Shuffle(IDENTITY); @Override @ForceInline - public Double128Vector toVector() { - return VectorSupport.shuffleToVector(VCLASS, ETYPE, Double128Shuffle.class, this, VLENGTH, - (s) -> ((Double128Vector)(((AbstractShuffle)(s)).toVectorTemplate()))); + Long128Vector toBitsVector() { + return (Long128Vector) super.toBitsVectorTemplate(); } @Override @ForceInline - public VectorShuffle cast(VectorSpecies s) { - AbstractSpecies species = (AbstractSpecies) s; - if (length() != species.laneCount()) - throw new IllegalArgumentException("VectorShuffle length and species length differ"); - int[] shuffleArray = toArray(); - return s.shuffleFromArray(shuffleArray, 0).check(s); + LongVector toBitsVector0() { + return Long128Vector.VSPECIES.dummyVector().vectorFactory(indices()); } + @Override @ForceInline + public int laneSource(int i) { + return (int)toBitsVector().lane(i); + } + @Override - public Double128Shuffle rearrange(VectorShuffle shuffle) { - Double128Shuffle s = (Double128Shuffle) shuffle; - byte[] reorder1 = reorder(); - byte[] reorder2 = s.reorder(); - byte[] r = new byte[reorder1.length]; - for (int i = 0; i < reorder1.length; i++) { - int ssi = reorder2[i]; - r[i] = reorder1[ssi]; // throws on exceptional index + @ForceInline + public void intoArray(int[] a, int offset) { + VectorSpecies species = VectorSpecies.of( + int.class, + VectorShape.forBitSize(length() * Integer.SIZE)); + Vector v = toBitsVector(); + v.convertShape(VectorOperators.L2I, species, 0) + .reinterpretAsInts() + .intoArray(a, offset); + } + + private static long[] prepare(int[] indices, int offset) { + long[] a = new long[VLENGTH]; + for (int i = 0; i < VLENGTH; i++) { + int si = indices[offset + i]; + si = partiallyWrapIndex(si, VLENGTH); + a[i] = (long)si; + } + return a; + } + + private static long[] prepare(IntUnaryOperator f) { + long[] a = new long[VLENGTH]; + for (int i = 0; i < VLENGTH; i++) { + int si = f.applyAsInt(i); + si = partiallyWrapIndex(si, VLENGTH); + a[i] = (long)si; + } + return a; + } + + private static boolean indicesInRange(long[] indices) { + int length = indices.length; + for (long si : indices) { + if (si >= (long)length || si < (long)(-length)) { + boolean assertsEnabled = false; + assert(assertsEnabled = true); + if (assertsEnabled) { + String msg = ("index "+si+"out of range ["+length+"] in "+ + java.util.Arrays.toString(indices)); + throw new AssertionError(msg); + } + return false; + } } - return new Double128Shuffle(r); + return true; } } diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double256Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double256Vector.java index 3f4ee1bdc041b..18325249d2ead 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double256Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double256Vector.java @@ -141,24 +141,9 @@ Double256Mask maskFromArray(boolean[] bits) { @ForceInline Double256Shuffle iotaShuffle() { return Double256Shuffle.IOTA; } - @ForceInline - Double256Shuffle iotaShuffle(int start, int step, boolean wrap) { - if (wrap) { - return (Double256Shuffle)VectorSupport.shuffleIota(ETYPE, Double256Shuffle.class, VSPECIES, VLENGTH, start, step, 1, - (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i*lstep + lstart, l)))); - } else { - return (Double256Shuffle)VectorSupport.shuffleIota(ETYPE, Double256Shuffle.class, VSPECIES, VLENGTH, start, step, 0, - (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (i*lstep + lstart))); - } - } - - @Override - @ForceInline - Double256Shuffle shuffleFromBytes(byte[] reorder) { return new Double256Shuffle(reorder); } - @Override @ForceInline - Double256Shuffle shuffleFromArray(int[] indexes, int i) { return new Double256Shuffle(indexes, i); } + Double256Shuffle shuffleFromArray(int[] indices, int i) { return new Double256Shuffle(indices, i); } @Override @ForceInline @@ -344,9 +329,11 @@ public final long reduceLanesToLong(VectorOperators.Associative op, return (long) super.reduceLanesTemplate(op, Double256Mask.class, (Double256Mask) m); // specialized } + @Override @ForceInline - public VectorShuffle toShuffle() { - return super.toShuffleTemplate(Double256Shuffle.class); // specialize + public final + VectorShuffle toShuffle(AbstractSpecies dsp) { + return super.toShuffleTemplate(dsp); } // Specialized unary testing @@ -775,25 +762,28 @@ static Double256Mask maskAll(boolean bit) { static final class Double256Shuffle extends AbstractShuffle { static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM - static final Class ETYPE = double.class; // used by the JVM + static final Class ETYPE = long.class; // used by the JVM - Double256Shuffle(byte[] reorder) { - super(VLENGTH, reorder); + Double256Shuffle(long[] indices) { + super(indices); + assert(VLENGTH == indices.length); + assert(indicesInRange(indices)); } - public Double256Shuffle(int[] reorder) { - super(VLENGTH, reorder); + Double256Shuffle(int[] indices, int i) { + this(prepare(indices, i)); } - public Double256Shuffle(int[] reorder, int i) { - super(VLENGTH, reorder, i); + Double256Shuffle(IntUnaryOperator fn) { + this(prepare(fn)); } - public Double256Shuffle(IntUnaryOperator fn) { - super(VLENGTH, fn); + long[] indices() { + return (long[])getPayload(); } @Override + @ForceInline public DoubleSpecies vspecies() { return VSPECIES; } @@ -801,40 +791,76 @@ public DoubleSpecies vspecies() { static { // There must be enough bits in the shuffle lanes to encode // VLENGTH valid indexes and VLENGTH exceptional ones. - assert(VLENGTH < Byte.MAX_VALUE); - assert(Byte.MIN_VALUE <= -VLENGTH); + assert(VLENGTH < Long.MAX_VALUE); + assert(Long.MIN_VALUE <= -VLENGTH); } static final Double256Shuffle IOTA = new Double256Shuffle(IDENTITY); @Override @ForceInline - public Double256Vector toVector() { - return VectorSupport.shuffleToVector(VCLASS, ETYPE, Double256Shuffle.class, this, VLENGTH, - (s) -> ((Double256Vector)(((AbstractShuffle)(s)).toVectorTemplate()))); + Long256Vector toBitsVector() { + return (Long256Vector) super.toBitsVectorTemplate(); } @Override @ForceInline - public VectorShuffle cast(VectorSpecies s) { - AbstractSpecies species = (AbstractSpecies) s; - if (length() != species.laneCount()) - throw new IllegalArgumentException("VectorShuffle length and species length differ"); - int[] shuffleArray = toArray(); - return s.shuffleFromArray(shuffleArray, 0).check(s); + LongVector toBitsVector0() { + return Long256Vector.VSPECIES.dummyVector().vectorFactory(indices()); } + @Override @ForceInline + public int laneSource(int i) { + return (int)toBitsVector().lane(i); + } + @Override - public Double256Shuffle rearrange(VectorShuffle shuffle) { - Double256Shuffle s = (Double256Shuffle) shuffle; - byte[] reorder1 = reorder(); - byte[] reorder2 = s.reorder(); - byte[] r = new byte[reorder1.length]; - for (int i = 0; i < reorder1.length; i++) { - int ssi = reorder2[i]; - r[i] = reorder1[ssi]; // throws on exceptional index + @ForceInline + public void intoArray(int[] a, int offset) { + VectorSpecies species = VectorSpecies.of( + int.class, + VectorShape.forBitSize(length() * Integer.SIZE)); + Vector v = toBitsVector(); + v.convertShape(VectorOperators.L2I, species, 0) + .reinterpretAsInts() + .intoArray(a, offset); + } + + private static long[] prepare(int[] indices, int offset) { + long[] a = new long[VLENGTH]; + for (int i = 0; i < VLENGTH; i++) { + int si = indices[offset + i]; + si = partiallyWrapIndex(si, VLENGTH); + a[i] = (long)si; + } + return a; + } + + private static long[] prepare(IntUnaryOperator f) { + long[] a = new long[VLENGTH]; + for (int i = 0; i < VLENGTH; i++) { + int si = f.applyAsInt(i); + si = partiallyWrapIndex(si, VLENGTH); + a[i] = (long)si; + } + return a; + } + + private static boolean indicesInRange(long[] indices) { + int length = indices.length; + for (long si : indices) { + if (si >= (long)length || si < (long)(-length)) { + boolean assertsEnabled = false; + assert(assertsEnabled = true); + if (assertsEnabled) { + String msg = ("index "+si+"out of range ["+length+"] in "+ + java.util.Arrays.toString(indices)); + throw new AssertionError(msg); + } + return false; + } } - return new Double256Shuffle(r); + return true; } } diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double512Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double512Vector.java index 3f8211eef27d9..3140fde3747b5 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double512Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double512Vector.java @@ -141,24 +141,9 @@ Double512Mask maskFromArray(boolean[] bits) { @ForceInline Double512Shuffle iotaShuffle() { return Double512Shuffle.IOTA; } - @ForceInline - Double512Shuffle iotaShuffle(int start, int step, boolean wrap) { - if (wrap) { - return (Double512Shuffle)VectorSupport.shuffleIota(ETYPE, Double512Shuffle.class, VSPECIES, VLENGTH, start, step, 1, - (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i*lstep + lstart, l)))); - } else { - return (Double512Shuffle)VectorSupport.shuffleIota(ETYPE, Double512Shuffle.class, VSPECIES, VLENGTH, start, step, 0, - (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (i*lstep + lstart))); - } - } - - @Override - @ForceInline - Double512Shuffle shuffleFromBytes(byte[] reorder) { return new Double512Shuffle(reorder); } - @Override @ForceInline - Double512Shuffle shuffleFromArray(int[] indexes, int i) { return new Double512Shuffle(indexes, i); } + Double512Shuffle shuffleFromArray(int[] indices, int i) { return new Double512Shuffle(indices, i); } @Override @ForceInline @@ -344,9 +329,11 @@ public final long reduceLanesToLong(VectorOperators.Associative op, return (long) super.reduceLanesTemplate(op, Double512Mask.class, (Double512Mask) m); // specialized } + @Override @ForceInline - public VectorShuffle toShuffle() { - return super.toShuffleTemplate(Double512Shuffle.class); // specialize + public final + VectorShuffle toShuffle(AbstractSpecies dsp) { + return super.toShuffleTemplate(dsp); } // Specialized unary testing @@ -783,25 +770,28 @@ static Double512Mask maskAll(boolean bit) { static final class Double512Shuffle extends AbstractShuffle { static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM - static final Class ETYPE = double.class; // used by the JVM + static final Class ETYPE = long.class; // used by the JVM - Double512Shuffle(byte[] reorder) { - super(VLENGTH, reorder); + Double512Shuffle(long[] indices) { + super(indices); + assert(VLENGTH == indices.length); + assert(indicesInRange(indices)); } - public Double512Shuffle(int[] reorder) { - super(VLENGTH, reorder); + Double512Shuffle(int[] indices, int i) { + this(prepare(indices, i)); } - public Double512Shuffle(int[] reorder, int i) { - super(VLENGTH, reorder, i); + Double512Shuffle(IntUnaryOperator fn) { + this(prepare(fn)); } - public Double512Shuffle(IntUnaryOperator fn) { - super(VLENGTH, fn); + long[] indices() { + return (long[])getPayload(); } @Override + @ForceInline public DoubleSpecies vspecies() { return VSPECIES; } @@ -809,40 +799,76 @@ public DoubleSpecies vspecies() { static { // There must be enough bits in the shuffle lanes to encode // VLENGTH valid indexes and VLENGTH exceptional ones. - assert(VLENGTH < Byte.MAX_VALUE); - assert(Byte.MIN_VALUE <= -VLENGTH); + assert(VLENGTH < Long.MAX_VALUE); + assert(Long.MIN_VALUE <= -VLENGTH); } static final Double512Shuffle IOTA = new Double512Shuffle(IDENTITY); @Override @ForceInline - public Double512Vector toVector() { - return VectorSupport.shuffleToVector(VCLASS, ETYPE, Double512Shuffle.class, this, VLENGTH, - (s) -> ((Double512Vector)(((AbstractShuffle)(s)).toVectorTemplate()))); + Long512Vector toBitsVector() { + return (Long512Vector) super.toBitsVectorTemplate(); } @Override @ForceInline - public VectorShuffle cast(VectorSpecies s) { - AbstractSpecies species = (AbstractSpecies) s; - if (length() != species.laneCount()) - throw new IllegalArgumentException("VectorShuffle length and species length differ"); - int[] shuffleArray = toArray(); - return s.shuffleFromArray(shuffleArray, 0).check(s); + LongVector toBitsVector0() { + return Long512Vector.VSPECIES.dummyVector().vectorFactory(indices()); } + @Override @ForceInline + public int laneSource(int i) { + return (int)toBitsVector().lane(i); + } + @Override - public Double512Shuffle rearrange(VectorShuffle shuffle) { - Double512Shuffle s = (Double512Shuffle) shuffle; - byte[] reorder1 = reorder(); - byte[] reorder2 = s.reorder(); - byte[] r = new byte[reorder1.length]; - for (int i = 0; i < reorder1.length; i++) { - int ssi = reorder2[i]; - r[i] = reorder1[ssi]; // throws on exceptional index + @ForceInline + public void intoArray(int[] a, int offset) { + VectorSpecies species = VectorSpecies.of( + int.class, + VectorShape.forBitSize(length() * Integer.SIZE)); + Vector v = toBitsVector(); + v.convertShape(VectorOperators.L2I, species, 0) + .reinterpretAsInts() + .intoArray(a, offset); + } + + private static long[] prepare(int[] indices, int offset) { + long[] a = new long[VLENGTH]; + for (int i = 0; i < VLENGTH; i++) { + int si = indices[offset + i]; + si = partiallyWrapIndex(si, VLENGTH); + a[i] = (long)si; + } + return a; + } + + private static long[] prepare(IntUnaryOperator f) { + long[] a = new long[VLENGTH]; + for (int i = 0; i < VLENGTH; i++) { + int si = f.applyAsInt(i); + si = partiallyWrapIndex(si, VLENGTH); + a[i] = (long)si; + } + return a; + } + + private static boolean indicesInRange(long[] indices) { + int length = indices.length; + for (long si : indices) { + if (si >= (long)length || si < (long)(-length)) { + boolean assertsEnabled = false; + assert(assertsEnabled = true); + if (assertsEnabled) { + String msg = ("index "+si+"out of range ["+length+"] in "+ + java.util.Arrays.toString(indices)); + throw new AssertionError(msg); + } + return false; + } } - return new Double512Shuffle(r); + return true; } } diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double64Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double64Vector.java index 9900c97f912d4..bce5b31a9b35f 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double64Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double64Vector.java @@ -141,24 +141,9 @@ Double64Mask maskFromArray(boolean[] bits) { @ForceInline Double64Shuffle iotaShuffle() { return Double64Shuffle.IOTA; } - @ForceInline - Double64Shuffle iotaShuffle(int start, int step, boolean wrap) { - if (wrap) { - return (Double64Shuffle)VectorSupport.shuffleIota(ETYPE, Double64Shuffle.class, VSPECIES, VLENGTH, start, step, 1, - (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i*lstep + lstart, l)))); - } else { - return (Double64Shuffle)VectorSupport.shuffleIota(ETYPE, Double64Shuffle.class, VSPECIES, VLENGTH, start, step, 0, - (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (i*lstep + lstart))); - } - } - - @Override - @ForceInline - Double64Shuffle shuffleFromBytes(byte[] reorder) { return new Double64Shuffle(reorder); } - @Override @ForceInline - Double64Shuffle shuffleFromArray(int[] indexes, int i) { return new Double64Shuffle(indexes, i); } + Double64Shuffle shuffleFromArray(int[] indices, int i) { return new Double64Shuffle(indices, i); } @Override @ForceInline @@ -344,9 +329,11 @@ public final long reduceLanesToLong(VectorOperators.Associative op, return (long) super.reduceLanesTemplate(op, Double64Mask.class, (Double64Mask) m); // specialized } + @Override @ForceInline - public VectorShuffle toShuffle() { - return super.toShuffleTemplate(Double64Shuffle.class); // specialize + public final + VectorShuffle toShuffle(AbstractSpecies dsp) { + return super.toShuffleTemplate(dsp); } // Specialized unary testing @@ -769,25 +756,28 @@ static Double64Mask maskAll(boolean bit) { static final class Double64Shuffle extends AbstractShuffle { static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM - static final Class ETYPE = double.class; // used by the JVM + static final Class ETYPE = long.class; // used by the JVM - Double64Shuffle(byte[] reorder) { - super(VLENGTH, reorder); + Double64Shuffle(long[] indices) { + super(indices); + assert(VLENGTH == indices.length); + assert(indicesInRange(indices)); } - public Double64Shuffle(int[] reorder) { - super(VLENGTH, reorder); + Double64Shuffle(int[] indices, int i) { + this(prepare(indices, i)); } - public Double64Shuffle(int[] reorder, int i) { - super(VLENGTH, reorder, i); + Double64Shuffle(IntUnaryOperator fn) { + this(prepare(fn)); } - public Double64Shuffle(IntUnaryOperator fn) { - super(VLENGTH, fn); + long[] indices() { + return (long[])getPayload(); } @Override + @ForceInline public DoubleSpecies vspecies() { return VSPECIES; } @@ -795,40 +785,70 @@ public DoubleSpecies vspecies() { static { // There must be enough bits in the shuffle lanes to encode // VLENGTH valid indexes and VLENGTH exceptional ones. - assert(VLENGTH < Byte.MAX_VALUE); - assert(Byte.MIN_VALUE <= -VLENGTH); + assert(VLENGTH < Long.MAX_VALUE); + assert(Long.MIN_VALUE <= -VLENGTH); } static final Double64Shuffle IOTA = new Double64Shuffle(IDENTITY); @Override @ForceInline - public Double64Vector toVector() { - return VectorSupport.shuffleToVector(VCLASS, ETYPE, Double64Shuffle.class, this, VLENGTH, - (s) -> ((Double64Vector)(((AbstractShuffle)(s)).toVectorTemplate()))); + Long64Vector toBitsVector() { + return (Long64Vector) super.toBitsVectorTemplate(); } @Override @ForceInline - public VectorShuffle cast(VectorSpecies s) { - AbstractSpecies species = (AbstractSpecies) s; - if (length() != species.laneCount()) - throw new IllegalArgumentException("VectorShuffle length and species length differ"); - int[] shuffleArray = toArray(); - return s.shuffleFromArray(shuffleArray, 0).check(s); + LongVector toBitsVector0() { + return Long64Vector.VSPECIES.dummyVector().vectorFactory(indices()); } + @Override @ForceInline + public int laneSource(int i) { + return (int)toBitsVector().lane(i); + } + @Override - public Double64Shuffle rearrange(VectorShuffle shuffle) { - Double64Shuffle s = (Double64Shuffle) shuffle; - byte[] reorder1 = reorder(); - byte[] reorder2 = s.reorder(); - byte[] r = new byte[reorder1.length]; - for (int i = 0; i < reorder1.length; i++) { - int ssi = reorder2[i]; - r[i] = reorder1[ssi]; // throws on exceptional index + @ForceInline + public void intoArray(int[] a, int offset) { + a[offset] = laneSource(0); + } + + private static long[] prepare(int[] indices, int offset) { + long[] a = new long[VLENGTH]; + for (int i = 0; i < VLENGTH; i++) { + int si = indices[offset + i]; + si = partiallyWrapIndex(si, VLENGTH); + a[i] = (long)si; + } + return a; + } + + private static long[] prepare(IntUnaryOperator f) { + long[] a = new long[VLENGTH]; + for (int i = 0; i < VLENGTH; i++) { + int si = f.applyAsInt(i); + si = partiallyWrapIndex(si, VLENGTH); + a[i] = (long)si; + } + return a; + } + + private static boolean indicesInRange(long[] indices) { + int length = indices.length; + for (long si : indices) { + if (si >= (long)length || si < (long)(-length)) { + boolean assertsEnabled = false; + assert(assertsEnabled = true); + if (assertsEnabled) { + String msg = ("index "+si+"out of range ["+length+"] in "+ + java.util.Arrays.toString(indices)); + throw new AssertionError(msg); + } + return false; + } } - return new Double64Shuffle(r); + return true; } } diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/DoubleMaxVector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/DoubleMaxVector.java index 32ff760fb3693..e1b24a64b8b8a 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/DoubleMaxVector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/DoubleMaxVector.java @@ -141,24 +141,9 @@ DoubleMaxMask maskFromArray(boolean[] bits) { @ForceInline DoubleMaxShuffle iotaShuffle() { return DoubleMaxShuffle.IOTA; } - @ForceInline - DoubleMaxShuffle iotaShuffle(int start, int step, boolean wrap) { - if (wrap) { - return (DoubleMaxShuffle)VectorSupport.shuffleIota(ETYPE, DoubleMaxShuffle.class, VSPECIES, VLENGTH, start, step, 1, - (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i*lstep + lstart, l)))); - } else { - return (DoubleMaxShuffle)VectorSupport.shuffleIota(ETYPE, DoubleMaxShuffle.class, VSPECIES, VLENGTH, start, step, 0, - (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (i*lstep + lstart))); - } - } - - @Override - @ForceInline - DoubleMaxShuffle shuffleFromBytes(byte[] reorder) { return new DoubleMaxShuffle(reorder); } - @Override @ForceInline - DoubleMaxShuffle shuffleFromArray(int[] indexes, int i) { return new DoubleMaxShuffle(indexes, i); } + DoubleMaxShuffle shuffleFromArray(int[] indices, int i) { return new DoubleMaxShuffle(indices, i); } @Override @ForceInline @@ -344,9 +329,11 @@ public final long reduceLanesToLong(VectorOperators.Associative op, return (long) super.reduceLanesTemplate(op, DoubleMaxMask.class, (DoubleMaxMask) m); // specialized } + @Override @ForceInline - public VectorShuffle toShuffle() { - return super.toShuffleTemplate(DoubleMaxShuffle.class); // specialize + public final + VectorShuffle toShuffle(AbstractSpecies dsp) { + return super.toShuffleTemplate(dsp); } // Specialized unary testing @@ -768,25 +755,28 @@ static DoubleMaxMask maskAll(boolean bit) { static final class DoubleMaxShuffle extends AbstractShuffle { static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM - static final Class ETYPE = double.class; // used by the JVM + static final Class ETYPE = long.class; // used by the JVM - DoubleMaxShuffle(byte[] reorder) { - super(VLENGTH, reorder); + DoubleMaxShuffle(long[] indices) { + super(indices); + assert(VLENGTH == indices.length); + assert(indicesInRange(indices)); } - public DoubleMaxShuffle(int[] reorder) { - super(VLENGTH, reorder); + DoubleMaxShuffle(int[] indices, int i) { + this(prepare(indices, i)); } - public DoubleMaxShuffle(int[] reorder, int i) { - super(VLENGTH, reorder, i); + DoubleMaxShuffle(IntUnaryOperator fn) { + this(prepare(fn)); } - public DoubleMaxShuffle(IntUnaryOperator fn) { - super(VLENGTH, fn); + long[] indices() { + return (long[])getPayload(); } @Override + @ForceInline public DoubleSpecies vspecies() { return VSPECIES; } @@ -794,40 +784,76 @@ public DoubleSpecies vspecies() { static { // There must be enough bits in the shuffle lanes to encode // VLENGTH valid indexes and VLENGTH exceptional ones. - assert(VLENGTH < Byte.MAX_VALUE); - assert(Byte.MIN_VALUE <= -VLENGTH); + assert(VLENGTH < Long.MAX_VALUE); + assert(Long.MIN_VALUE <= -VLENGTH); } static final DoubleMaxShuffle IOTA = new DoubleMaxShuffle(IDENTITY); @Override @ForceInline - public DoubleMaxVector toVector() { - return VectorSupport.shuffleToVector(VCLASS, ETYPE, DoubleMaxShuffle.class, this, VLENGTH, - (s) -> ((DoubleMaxVector)(((AbstractShuffle)(s)).toVectorTemplate()))); + LongMaxVector toBitsVector() { + return (LongMaxVector) super.toBitsVectorTemplate(); } @Override @ForceInline - public VectorShuffle cast(VectorSpecies s) { - AbstractSpecies species = (AbstractSpecies) s; - if (length() != species.laneCount()) - throw new IllegalArgumentException("VectorShuffle length and species length differ"); - int[] shuffleArray = toArray(); - return s.shuffleFromArray(shuffleArray, 0).check(s); + LongVector toBitsVector0() { + return LongMaxVector.VSPECIES.dummyVector().vectorFactory(indices()); } + @Override @ForceInline + public int laneSource(int i) { + return (int)toBitsVector().lane(i); + } + @Override - public DoubleMaxShuffle rearrange(VectorShuffle shuffle) { - DoubleMaxShuffle s = (DoubleMaxShuffle) shuffle; - byte[] reorder1 = reorder(); - byte[] reorder2 = s.reorder(); - byte[] r = new byte[reorder1.length]; - for (int i = 0; i < reorder1.length; i++) { - int ssi = reorder2[i]; - r[i] = reorder1[ssi]; // throws on exceptional index + @ForceInline + public void intoArray(int[] a, int offset) { + VectorSpecies species = VectorSpecies.of( + int.class, + VectorShape.forBitSize(length() * Integer.SIZE)); + Vector v = toBitsVector(); + v.convertShape(VectorOperators.L2I, species, 0) + .reinterpretAsInts() + .intoArray(a, offset); + } + + private static long[] prepare(int[] indices, int offset) { + long[] a = new long[VLENGTH]; + for (int i = 0; i < VLENGTH; i++) { + int si = indices[offset + i]; + si = partiallyWrapIndex(si, VLENGTH); + a[i] = (long)si; + } + return a; + } + + private static long[] prepare(IntUnaryOperator f) { + long[] a = new long[VLENGTH]; + for (int i = 0; i < VLENGTH; i++) { + int si = f.applyAsInt(i); + si = partiallyWrapIndex(si, VLENGTH); + a[i] = (long)si; + } + return a; + } + + private static boolean indicesInRange(long[] indices) { + int length = indices.length; + for (long si : indices) { + if (si >= (long)length || si < (long)(-length)) { + boolean assertsEnabled = false; + assert(assertsEnabled = true); + if (assertsEnabled) { + String msg = ("index "+si+"out of range ["+length+"] in "+ + java.util.Arrays.toString(indices)); + throw new AssertionError(msg); + } + return false; + } } - return new DoubleMaxShuffle(r); + return true; } } diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/DoubleVector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/DoubleVector.java index 64bd795f817d0..1f4c0a49d45c2 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/DoubleVector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/DoubleVector.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, 2022, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2017, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -953,7 +953,7 @@ DoubleVector lanewise(VectorOperators.Binary op, // and broadcast, but it would be more surprising not to continue // the obvious pattern started by unary and binary. - /** + /** * {@inheritDoc} * @see #lanewise(VectorOperators.Ternary,double,double,VectorMask) * @see #lanewise(VectorOperators.Ternary,Vector,double,VectorMask) @@ -2322,8 +2322,8 @@ DoubleVector rearrangeTemplate(Class shuffletype, } @ForceInline - private final - VectorShuffle toShuffle0(DoubleSpecies dsp) { + final + VectorShuffle toShuffle0(AbstractSpecies dsp) { double[] a = toArray(); int[] sa = new int[a.length]; for (int i = 0; i < a.length; i++) { @@ -2332,18 +2332,6 @@ VectorShuffle toShuffle0(DoubleSpecies dsp) { return VectorShuffle.fromArray(dsp, sa, 0); } - /*package-private*/ - @ForceInline - final - VectorShuffle toShuffleTemplate(Class shuffleType) { - DoubleSpecies vsp = vspecies(); - return VectorSupport.convert(VectorSupport.VECTOR_OP_CAST, - getClass(), double.class, length(), - shuffleType, byte.class, length(), - this, vsp, - DoubleVector::toShuffle0); - } - /** * {@inheritDoc} * @since 19 @@ -3699,9 +3687,10 @@ static final class DoubleSpecies extends AbstractSpecies { private DoubleSpecies(VectorShape shape, Class vectorType, Class> maskType, + Class> shuffleType, Function vectorFactory) { super(shape, LaneType.of(double.class), - vectorType, maskType, + vectorType, maskType, shuffleType, vectorFactory); assert(this.elementSize() == Double.SIZE); } @@ -3977,6 +3966,7 @@ static DoubleSpecies species(VectorShape s) { = new DoubleSpecies(VectorShape.S_64_BIT, Double64Vector.class, Double64Vector.Double64Mask.class, + Double64Vector.Double64Shuffle.class, Double64Vector::new); /** Species representing {@link DoubleVector}s of {@link VectorShape#S_128_BIT VectorShape.S_128_BIT}. */ @@ -3984,6 +3974,7 @@ static DoubleSpecies species(VectorShape s) { = new DoubleSpecies(VectorShape.S_128_BIT, Double128Vector.class, Double128Vector.Double128Mask.class, + Double128Vector.Double128Shuffle.class, Double128Vector::new); /** Species representing {@link DoubleVector}s of {@link VectorShape#S_256_BIT VectorShape.S_256_BIT}. */ @@ -3991,6 +3982,7 @@ static DoubleSpecies species(VectorShape s) { = new DoubleSpecies(VectorShape.S_256_BIT, Double256Vector.class, Double256Vector.Double256Mask.class, + Double256Vector.Double256Shuffle.class, Double256Vector::new); /** Species representing {@link DoubleVector}s of {@link VectorShape#S_512_BIT VectorShape.S_512_BIT}. */ @@ -3998,6 +3990,7 @@ static DoubleSpecies species(VectorShape s) { = new DoubleSpecies(VectorShape.S_512_BIT, Double512Vector.class, Double512Vector.Double512Mask.class, + Double512Vector.Double512Shuffle.class, Double512Vector::new); /** Species representing {@link DoubleVector}s of {@link VectorShape#S_Max_BIT VectorShape.S_Max_BIT}. */ @@ -4005,6 +3998,7 @@ static DoubleSpecies species(VectorShape s) { = new DoubleSpecies(VectorShape.S_Max_BIT, DoubleMaxVector.class, DoubleMaxVector.DoubleMaxMask.class, + DoubleMaxVector.DoubleMaxShuffle.class, DoubleMaxVector::new); /** diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float128Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float128Vector.java index 5414be75607bf..7ccf752540a83 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float128Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float128Vector.java @@ -141,24 +141,9 @@ Float128Mask maskFromArray(boolean[] bits) { @ForceInline Float128Shuffle iotaShuffle() { return Float128Shuffle.IOTA; } - @ForceInline - Float128Shuffle iotaShuffle(int start, int step, boolean wrap) { - if (wrap) { - return (Float128Shuffle)VectorSupport.shuffleIota(ETYPE, Float128Shuffle.class, VSPECIES, VLENGTH, start, step, 1, - (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i*lstep + lstart, l)))); - } else { - return (Float128Shuffle)VectorSupport.shuffleIota(ETYPE, Float128Shuffle.class, VSPECIES, VLENGTH, start, step, 0, - (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (i*lstep + lstart))); - } - } - - @Override - @ForceInline - Float128Shuffle shuffleFromBytes(byte[] reorder) { return new Float128Shuffle(reorder); } - @Override @ForceInline - Float128Shuffle shuffleFromArray(int[] indexes, int i) { return new Float128Shuffle(indexes, i); } + Float128Shuffle shuffleFromArray(int[] indices, int i) { return new Float128Shuffle(indices, i); } @Override @ForceInline @@ -344,9 +329,11 @@ public final long reduceLanesToLong(VectorOperators.Associative op, return (long) super.reduceLanesTemplate(op, Float128Mask.class, (Float128Mask) m); // specialized } + @Override @ForceInline - public VectorShuffle toShuffle() { - return super.toShuffleTemplate(Float128Shuffle.class); // specialize + public final + VectorShuffle toShuffle(AbstractSpecies dsp) { + return super.toShuffleTemplate(dsp); } // Specialized unary testing @@ -775,25 +762,28 @@ static Float128Mask maskAll(boolean bit) { static final class Float128Shuffle extends AbstractShuffle { static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM - static final Class ETYPE = float.class; // used by the JVM + static final Class ETYPE = int.class; // used by the JVM - Float128Shuffle(byte[] reorder) { - super(VLENGTH, reorder); + Float128Shuffle(int[] indices) { + super(indices); + assert(VLENGTH == indices.length); + assert(indicesInRange(indices)); } - public Float128Shuffle(int[] reorder) { - super(VLENGTH, reorder); + Float128Shuffle(int[] indices, int i) { + this(prepare(indices, i)); } - public Float128Shuffle(int[] reorder, int i) { - super(VLENGTH, reorder, i); + Float128Shuffle(IntUnaryOperator fn) { + this(prepare(fn)); } - public Float128Shuffle(IntUnaryOperator fn) { - super(VLENGTH, fn); + int[] indices() { + return (int[])getPayload(); } @Override + @ForceInline public FloatSpecies vspecies() { return VSPECIES; } @@ -801,40 +791,70 @@ public FloatSpecies vspecies() { static { // There must be enough bits in the shuffle lanes to encode // VLENGTH valid indexes and VLENGTH exceptional ones. - assert(VLENGTH < Byte.MAX_VALUE); - assert(Byte.MIN_VALUE <= -VLENGTH); + assert(VLENGTH < Integer.MAX_VALUE); + assert(Integer.MIN_VALUE <= -VLENGTH); } static final Float128Shuffle IOTA = new Float128Shuffle(IDENTITY); @Override @ForceInline - public Float128Vector toVector() { - return VectorSupport.shuffleToVector(VCLASS, ETYPE, Float128Shuffle.class, this, VLENGTH, - (s) -> ((Float128Vector)(((AbstractShuffle)(s)).toVectorTemplate()))); + Int128Vector toBitsVector() { + return (Int128Vector) super.toBitsVectorTemplate(); } @Override @ForceInline - public VectorShuffle cast(VectorSpecies s) { - AbstractSpecies species = (AbstractSpecies) s; - if (length() != species.laneCount()) - throw new IllegalArgumentException("VectorShuffle length and species length differ"); - int[] shuffleArray = toArray(); - return s.shuffleFromArray(shuffleArray, 0).check(s); + IntVector toBitsVector0() { + return Int128Vector.VSPECIES.dummyVector().vectorFactory(indices()); } + @Override @ForceInline + public int laneSource(int i) { + return (int)toBitsVector().lane(i); + } + @Override - public Float128Shuffle rearrange(VectorShuffle shuffle) { - Float128Shuffle s = (Float128Shuffle) shuffle; - byte[] reorder1 = reorder(); - byte[] reorder2 = s.reorder(); - byte[] r = new byte[reorder1.length]; - for (int i = 0; i < reorder1.length; i++) { - int ssi = reorder2[i]; - r[i] = reorder1[ssi]; // throws on exceptional index + @ForceInline + public void intoArray(int[] a, int offset) { + toBitsVector().intoArray(a, offset); + } + + private static int[] prepare(int[] indices, int offset) { + int[] a = new int[VLENGTH]; + for (int i = 0; i < VLENGTH; i++) { + int si = indices[offset + i]; + si = partiallyWrapIndex(si, VLENGTH); + a[i] = (int)si; + } + return a; + } + + private static int[] prepare(IntUnaryOperator f) { + int[] a = new int[VLENGTH]; + for (int i = 0; i < VLENGTH; i++) { + int si = f.applyAsInt(i); + si = partiallyWrapIndex(si, VLENGTH); + a[i] = (int)si; + } + return a; + } + + private static boolean indicesInRange(int[] indices) { + int length = indices.length; + for (int si : indices) { + if (si >= (int)length || si < (int)(-length)) { + boolean assertsEnabled = false; + assert(assertsEnabled = true); + if (assertsEnabled) { + String msg = ("index "+si+"out of range ["+length+"] in "+ + java.util.Arrays.toString(indices)); + throw new AssertionError(msg); + } + return false; + } } - return new Float128Shuffle(r); + return true; } } diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float256Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float256Vector.java index 9194fbcde0edb..fd1763b0c40e2 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float256Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float256Vector.java @@ -141,24 +141,9 @@ Float256Mask maskFromArray(boolean[] bits) { @ForceInline Float256Shuffle iotaShuffle() { return Float256Shuffle.IOTA; } - @ForceInline - Float256Shuffle iotaShuffle(int start, int step, boolean wrap) { - if (wrap) { - return (Float256Shuffle)VectorSupport.shuffleIota(ETYPE, Float256Shuffle.class, VSPECIES, VLENGTH, start, step, 1, - (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i*lstep + lstart, l)))); - } else { - return (Float256Shuffle)VectorSupport.shuffleIota(ETYPE, Float256Shuffle.class, VSPECIES, VLENGTH, start, step, 0, - (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (i*lstep + lstart))); - } - } - - @Override - @ForceInline - Float256Shuffle shuffleFromBytes(byte[] reorder) { return new Float256Shuffle(reorder); } - @Override @ForceInline - Float256Shuffle shuffleFromArray(int[] indexes, int i) { return new Float256Shuffle(indexes, i); } + Float256Shuffle shuffleFromArray(int[] indices, int i) { return new Float256Shuffle(indices, i); } @Override @ForceInline @@ -344,9 +329,11 @@ public final long reduceLanesToLong(VectorOperators.Associative op, return (long) super.reduceLanesTemplate(op, Float256Mask.class, (Float256Mask) m); // specialized } + @Override @ForceInline - public VectorShuffle toShuffle() { - return super.toShuffleTemplate(Float256Shuffle.class); // specialize + public final + VectorShuffle toShuffle(AbstractSpecies dsp) { + return super.toShuffleTemplate(dsp); } // Specialized unary testing @@ -783,25 +770,28 @@ static Float256Mask maskAll(boolean bit) { static final class Float256Shuffle extends AbstractShuffle { static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM - static final Class ETYPE = float.class; // used by the JVM + static final Class ETYPE = int.class; // used by the JVM - Float256Shuffle(byte[] reorder) { - super(VLENGTH, reorder); + Float256Shuffle(int[] indices) { + super(indices); + assert(VLENGTH == indices.length); + assert(indicesInRange(indices)); } - public Float256Shuffle(int[] reorder) { - super(VLENGTH, reorder); + Float256Shuffle(int[] indices, int i) { + this(prepare(indices, i)); } - public Float256Shuffle(int[] reorder, int i) { - super(VLENGTH, reorder, i); + Float256Shuffle(IntUnaryOperator fn) { + this(prepare(fn)); } - public Float256Shuffle(IntUnaryOperator fn) { - super(VLENGTH, fn); + int[] indices() { + return (int[])getPayload(); } @Override + @ForceInline public FloatSpecies vspecies() { return VSPECIES; } @@ -809,40 +799,70 @@ public FloatSpecies vspecies() { static { // There must be enough bits in the shuffle lanes to encode // VLENGTH valid indexes and VLENGTH exceptional ones. - assert(VLENGTH < Byte.MAX_VALUE); - assert(Byte.MIN_VALUE <= -VLENGTH); + assert(VLENGTH < Integer.MAX_VALUE); + assert(Integer.MIN_VALUE <= -VLENGTH); } static final Float256Shuffle IOTA = new Float256Shuffle(IDENTITY); @Override @ForceInline - public Float256Vector toVector() { - return VectorSupport.shuffleToVector(VCLASS, ETYPE, Float256Shuffle.class, this, VLENGTH, - (s) -> ((Float256Vector)(((AbstractShuffle)(s)).toVectorTemplate()))); + Int256Vector toBitsVector() { + return (Int256Vector) super.toBitsVectorTemplate(); } @Override @ForceInline - public VectorShuffle cast(VectorSpecies s) { - AbstractSpecies species = (AbstractSpecies) s; - if (length() != species.laneCount()) - throw new IllegalArgumentException("VectorShuffle length and species length differ"); - int[] shuffleArray = toArray(); - return s.shuffleFromArray(shuffleArray, 0).check(s); + IntVector toBitsVector0() { + return Int256Vector.VSPECIES.dummyVector().vectorFactory(indices()); } + @Override @ForceInline + public int laneSource(int i) { + return (int)toBitsVector().lane(i); + } + @Override - public Float256Shuffle rearrange(VectorShuffle shuffle) { - Float256Shuffle s = (Float256Shuffle) shuffle; - byte[] reorder1 = reorder(); - byte[] reorder2 = s.reorder(); - byte[] r = new byte[reorder1.length]; - for (int i = 0; i < reorder1.length; i++) { - int ssi = reorder2[i]; - r[i] = reorder1[ssi]; // throws on exceptional index + @ForceInline + public void intoArray(int[] a, int offset) { + toBitsVector().intoArray(a, offset); + } + + private static int[] prepare(int[] indices, int offset) { + int[] a = new int[VLENGTH]; + for (int i = 0; i < VLENGTH; i++) { + int si = indices[offset + i]; + si = partiallyWrapIndex(si, VLENGTH); + a[i] = (int)si; + } + return a; + } + + private static int[] prepare(IntUnaryOperator f) { + int[] a = new int[VLENGTH]; + for (int i = 0; i < VLENGTH; i++) { + int si = f.applyAsInt(i); + si = partiallyWrapIndex(si, VLENGTH); + a[i] = (int)si; + } + return a; + } + + private static boolean indicesInRange(int[] indices) { + int length = indices.length; + for (int si : indices) { + if (si >= (int)length || si < (int)(-length)) { + boolean assertsEnabled = false; + assert(assertsEnabled = true); + if (assertsEnabled) { + String msg = ("index "+si+"out of range ["+length+"] in "+ + java.util.Arrays.toString(indices)); + throw new AssertionError(msg); + } + return false; + } } - return new Float256Shuffle(r); + return true; } } diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float512Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float512Vector.java index 2e43f153b0c55..03dfb1e5049f9 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float512Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float512Vector.java @@ -141,24 +141,9 @@ Float512Mask maskFromArray(boolean[] bits) { @ForceInline Float512Shuffle iotaShuffle() { return Float512Shuffle.IOTA; } - @ForceInline - Float512Shuffle iotaShuffle(int start, int step, boolean wrap) { - if (wrap) { - return (Float512Shuffle)VectorSupport.shuffleIota(ETYPE, Float512Shuffle.class, VSPECIES, VLENGTH, start, step, 1, - (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i*lstep + lstart, l)))); - } else { - return (Float512Shuffle)VectorSupport.shuffleIota(ETYPE, Float512Shuffle.class, VSPECIES, VLENGTH, start, step, 0, - (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (i*lstep + lstart))); - } - } - - @Override - @ForceInline - Float512Shuffle shuffleFromBytes(byte[] reorder) { return new Float512Shuffle(reorder); } - @Override @ForceInline - Float512Shuffle shuffleFromArray(int[] indexes, int i) { return new Float512Shuffle(indexes, i); } + Float512Shuffle shuffleFromArray(int[] indices, int i) { return new Float512Shuffle(indices, i); } @Override @ForceInline @@ -344,9 +329,11 @@ public final long reduceLanesToLong(VectorOperators.Associative op, return (long) super.reduceLanesTemplate(op, Float512Mask.class, (Float512Mask) m); // specialized } + @Override @ForceInline - public VectorShuffle toShuffle() { - return super.toShuffleTemplate(Float512Shuffle.class); // specialize + public final + VectorShuffle toShuffle(AbstractSpecies dsp) { + return super.toShuffleTemplate(dsp); } // Specialized unary testing @@ -799,25 +786,28 @@ static Float512Mask maskAll(boolean bit) { static final class Float512Shuffle extends AbstractShuffle { static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM - static final Class ETYPE = float.class; // used by the JVM + static final Class ETYPE = int.class; // used by the JVM - Float512Shuffle(byte[] reorder) { - super(VLENGTH, reorder); + Float512Shuffle(int[] indices) { + super(indices); + assert(VLENGTH == indices.length); + assert(indicesInRange(indices)); } - public Float512Shuffle(int[] reorder) { - super(VLENGTH, reorder); + Float512Shuffle(int[] indices, int i) { + this(prepare(indices, i)); } - public Float512Shuffle(int[] reorder, int i) { - super(VLENGTH, reorder, i); + Float512Shuffle(IntUnaryOperator fn) { + this(prepare(fn)); } - public Float512Shuffle(IntUnaryOperator fn) { - super(VLENGTH, fn); + int[] indices() { + return (int[])getPayload(); } @Override + @ForceInline public FloatSpecies vspecies() { return VSPECIES; } @@ -825,40 +815,70 @@ public FloatSpecies vspecies() { static { // There must be enough bits in the shuffle lanes to encode // VLENGTH valid indexes and VLENGTH exceptional ones. - assert(VLENGTH < Byte.MAX_VALUE); - assert(Byte.MIN_VALUE <= -VLENGTH); + assert(VLENGTH < Integer.MAX_VALUE); + assert(Integer.MIN_VALUE <= -VLENGTH); } static final Float512Shuffle IOTA = new Float512Shuffle(IDENTITY); @Override @ForceInline - public Float512Vector toVector() { - return VectorSupport.shuffleToVector(VCLASS, ETYPE, Float512Shuffle.class, this, VLENGTH, - (s) -> ((Float512Vector)(((AbstractShuffle)(s)).toVectorTemplate()))); + Int512Vector toBitsVector() { + return (Int512Vector) super.toBitsVectorTemplate(); } @Override @ForceInline - public VectorShuffle cast(VectorSpecies s) { - AbstractSpecies species = (AbstractSpecies) s; - if (length() != species.laneCount()) - throw new IllegalArgumentException("VectorShuffle length and species length differ"); - int[] shuffleArray = toArray(); - return s.shuffleFromArray(shuffleArray, 0).check(s); + IntVector toBitsVector0() { + return Int512Vector.VSPECIES.dummyVector().vectorFactory(indices()); } + @Override @ForceInline + public int laneSource(int i) { + return (int)toBitsVector().lane(i); + } + @Override - public Float512Shuffle rearrange(VectorShuffle shuffle) { - Float512Shuffle s = (Float512Shuffle) shuffle; - byte[] reorder1 = reorder(); - byte[] reorder2 = s.reorder(); - byte[] r = new byte[reorder1.length]; - for (int i = 0; i < reorder1.length; i++) { - int ssi = reorder2[i]; - r[i] = reorder1[ssi]; // throws on exceptional index + @ForceInline + public void intoArray(int[] a, int offset) { + toBitsVector().intoArray(a, offset); + } + + private static int[] prepare(int[] indices, int offset) { + int[] a = new int[VLENGTH]; + for (int i = 0; i < VLENGTH; i++) { + int si = indices[offset + i]; + si = partiallyWrapIndex(si, VLENGTH); + a[i] = (int)si; + } + return a; + } + + private static int[] prepare(IntUnaryOperator f) { + int[] a = new int[VLENGTH]; + for (int i = 0; i < VLENGTH; i++) { + int si = f.applyAsInt(i); + si = partiallyWrapIndex(si, VLENGTH); + a[i] = (int)si; + } + return a; + } + + private static boolean indicesInRange(int[] indices) { + int length = indices.length; + for (int si : indices) { + if (si >= (int)length || si < (int)(-length)) { + boolean assertsEnabled = false; + assert(assertsEnabled = true); + if (assertsEnabled) { + String msg = ("index "+si+"out of range ["+length+"] in "+ + java.util.Arrays.toString(indices)); + throw new AssertionError(msg); + } + return false; + } } - return new Float512Shuffle(r); + return true; } } diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float64Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float64Vector.java index ccd75e18e062a..48d76adb3ebb1 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float64Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float64Vector.java @@ -141,24 +141,9 @@ Float64Mask maskFromArray(boolean[] bits) { @ForceInline Float64Shuffle iotaShuffle() { return Float64Shuffle.IOTA; } - @ForceInline - Float64Shuffle iotaShuffle(int start, int step, boolean wrap) { - if (wrap) { - return (Float64Shuffle)VectorSupport.shuffleIota(ETYPE, Float64Shuffle.class, VSPECIES, VLENGTH, start, step, 1, - (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i*lstep + lstart, l)))); - } else { - return (Float64Shuffle)VectorSupport.shuffleIota(ETYPE, Float64Shuffle.class, VSPECIES, VLENGTH, start, step, 0, - (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (i*lstep + lstart))); - } - } - - @Override - @ForceInline - Float64Shuffle shuffleFromBytes(byte[] reorder) { return new Float64Shuffle(reorder); } - @Override @ForceInline - Float64Shuffle shuffleFromArray(int[] indexes, int i) { return new Float64Shuffle(indexes, i); } + Float64Shuffle shuffleFromArray(int[] indices, int i) { return new Float64Shuffle(indices, i); } @Override @ForceInline @@ -344,9 +329,11 @@ public final long reduceLanesToLong(VectorOperators.Associative op, return (long) super.reduceLanesTemplate(op, Float64Mask.class, (Float64Mask) m); // specialized } + @Override @ForceInline - public VectorShuffle toShuffle() { - return super.toShuffleTemplate(Float64Shuffle.class); // specialize + public final + VectorShuffle toShuffle(AbstractSpecies dsp) { + return super.toShuffleTemplate(dsp); } // Specialized unary testing @@ -771,25 +758,28 @@ static Float64Mask maskAll(boolean bit) { static final class Float64Shuffle extends AbstractShuffle { static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM - static final Class ETYPE = float.class; // used by the JVM + static final Class ETYPE = int.class; // used by the JVM - Float64Shuffle(byte[] reorder) { - super(VLENGTH, reorder); + Float64Shuffle(int[] indices) { + super(indices); + assert(VLENGTH == indices.length); + assert(indicesInRange(indices)); } - public Float64Shuffle(int[] reorder) { - super(VLENGTH, reorder); + Float64Shuffle(int[] indices, int i) { + this(prepare(indices, i)); } - public Float64Shuffle(int[] reorder, int i) { - super(VLENGTH, reorder, i); + Float64Shuffle(IntUnaryOperator fn) { + this(prepare(fn)); } - public Float64Shuffle(IntUnaryOperator fn) { - super(VLENGTH, fn); + int[] indices() { + return (int[])getPayload(); } @Override + @ForceInline public FloatSpecies vspecies() { return VSPECIES; } @@ -797,40 +787,70 @@ public FloatSpecies vspecies() { static { // There must be enough bits in the shuffle lanes to encode // VLENGTH valid indexes and VLENGTH exceptional ones. - assert(VLENGTH < Byte.MAX_VALUE); - assert(Byte.MIN_VALUE <= -VLENGTH); + assert(VLENGTH < Integer.MAX_VALUE); + assert(Integer.MIN_VALUE <= -VLENGTH); } static final Float64Shuffle IOTA = new Float64Shuffle(IDENTITY); @Override @ForceInline - public Float64Vector toVector() { - return VectorSupport.shuffleToVector(VCLASS, ETYPE, Float64Shuffle.class, this, VLENGTH, - (s) -> ((Float64Vector)(((AbstractShuffle)(s)).toVectorTemplate()))); + Int64Vector toBitsVector() { + return (Int64Vector) super.toBitsVectorTemplate(); } @Override @ForceInline - public VectorShuffle cast(VectorSpecies s) { - AbstractSpecies species = (AbstractSpecies) s; - if (length() != species.laneCount()) - throw new IllegalArgumentException("VectorShuffle length and species length differ"); - int[] shuffleArray = toArray(); - return s.shuffleFromArray(shuffleArray, 0).check(s); + IntVector toBitsVector0() { + return Int64Vector.VSPECIES.dummyVector().vectorFactory(indices()); } + @Override @ForceInline + public int laneSource(int i) { + return (int)toBitsVector().lane(i); + } + @Override - public Float64Shuffle rearrange(VectorShuffle shuffle) { - Float64Shuffle s = (Float64Shuffle) shuffle; - byte[] reorder1 = reorder(); - byte[] reorder2 = s.reorder(); - byte[] r = new byte[reorder1.length]; - for (int i = 0; i < reorder1.length; i++) { - int ssi = reorder2[i]; - r[i] = reorder1[ssi]; // throws on exceptional index + @ForceInline + public void intoArray(int[] a, int offset) { + toBitsVector().intoArray(a, offset); + } + + private static int[] prepare(int[] indices, int offset) { + int[] a = new int[VLENGTH]; + for (int i = 0; i < VLENGTH; i++) { + int si = indices[offset + i]; + si = partiallyWrapIndex(si, VLENGTH); + a[i] = (int)si; + } + return a; + } + + private static int[] prepare(IntUnaryOperator f) { + int[] a = new int[VLENGTH]; + for (int i = 0; i < VLENGTH; i++) { + int si = f.applyAsInt(i); + si = partiallyWrapIndex(si, VLENGTH); + a[i] = (int)si; + } + return a; + } + + private static boolean indicesInRange(int[] indices) { + int length = indices.length; + for (int si : indices) { + if (si >= (int)length || si < (int)(-length)) { + boolean assertsEnabled = false; + assert(assertsEnabled = true); + if (assertsEnabled) { + String msg = ("index "+si+"out of range ["+length+"] in "+ + java.util.Arrays.toString(indices)); + throw new AssertionError(msg); + } + return false; + } } - return new Float64Shuffle(r); + return true; } } diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/FloatMaxVector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/FloatMaxVector.java index e5b4651531b45..ffea6b909e135 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/FloatMaxVector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/FloatMaxVector.java @@ -141,24 +141,9 @@ FloatMaxMask maskFromArray(boolean[] bits) { @ForceInline FloatMaxShuffle iotaShuffle() { return FloatMaxShuffle.IOTA; } - @ForceInline - FloatMaxShuffle iotaShuffle(int start, int step, boolean wrap) { - if (wrap) { - return (FloatMaxShuffle)VectorSupport.shuffleIota(ETYPE, FloatMaxShuffle.class, VSPECIES, VLENGTH, start, step, 1, - (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i*lstep + lstart, l)))); - } else { - return (FloatMaxShuffle)VectorSupport.shuffleIota(ETYPE, FloatMaxShuffle.class, VSPECIES, VLENGTH, start, step, 0, - (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (i*lstep + lstart))); - } - } - - @Override - @ForceInline - FloatMaxShuffle shuffleFromBytes(byte[] reorder) { return new FloatMaxShuffle(reorder); } - @Override @ForceInline - FloatMaxShuffle shuffleFromArray(int[] indexes, int i) { return new FloatMaxShuffle(indexes, i); } + FloatMaxShuffle shuffleFromArray(int[] indices, int i) { return new FloatMaxShuffle(indices, i); } @Override @ForceInline @@ -344,9 +329,11 @@ public final long reduceLanesToLong(VectorOperators.Associative op, return (long) super.reduceLanesTemplate(op, FloatMaxMask.class, (FloatMaxMask) m); // specialized } + @Override @ForceInline - public VectorShuffle toShuffle() { - return super.toShuffleTemplate(FloatMaxShuffle.class); // specialize + public final + VectorShuffle toShuffle(AbstractSpecies dsp) { + return super.toShuffleTemplate(dsp); } // Specialized unary testing @@ -768,25 +755,28 @@ static FloatMaxMask maskAll(boolean bit) { static final class FloatMaxShuffle extends AbstractShuffle { static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM - static final Class ETYPE = float.class; // used by the JVM + static final Class ETYPE = int.class; // used by the JVM - FloatMaxShuffle(byte[] reorder) { - super(VLENGTH, reorder); + FloatMaxShuffle(int[] indices) { + super(indices); + assert(VLENGTH == indices.length); + assert(indicesInRange(indices)); } - public FloatMaxShuffle(int[] reorder) { - super(VLENGTH, reorder); + FloatMaxShuffle(int[] indices, int i) { + this(prepare(indices, i)); } - public FloatMaxShuffle(int[] reorder, int i) { - super(VLENGTH, reorder, i); + FloatMaxShuffle(IntUnaryOperator fn) { + this(prepare(fn)); } - public FloatMaxShuffle(IntUnaryOperator fn) { - super(VLENGTH, fn); + int[] indices() { + return (int[])getPayload(); } @Override + @ForceInline public FloatSpecies vspecies() { return VSPECIES; } @@ -794,40 +784,70 @@ public FloatSpecies vspecies() { static { // There must be enough bits in the shuffle lanes to encode // VLENGTH valid indexes and VLENGTH exceptional ones. - assert(VLENGTH < Byte.MAX_VALUE); - assert(Byte.MIN_VALUE <= -VLENGTH); + assert(VLENGTH < Integer.MAX_VALUE); + assert(Integer.MIN_VALUE <= -VLENGTH); } static final FloatMaxShuffle IOTA = new FloatMaxShuffle(IDENTITY); @Override @ForceInline - public FloatMaxVector toVector() { - return VectorSupport.shuffleToVector(VCLASS, ETYPE, FloatMaxShuffle.class, this, VLENGTH, - (s) -> ((FloatMaxVector)(((AbstractShuffle)(s)).toVectorTemplate()))); + IntMaxVector toBitsVector() { + return (IntMaxVector) super.toBitsVectorTemplate(); } @Override @ForceInline - public VectorShuffle cast(VectorSpecies s) { - AbstractSpecies species = (AbstractSpecies) s; - if (length() != species.laneCount()) - throw new IllegalArgumentException("VectorShuffle length and species length differ"); - int[] shuffleArray = toArray(); - return s.shuffleFromArray(shuffleArray, 0).check(s); + IntVector toBitsVector0() { + return IntMaxVector.VSPECIES.dummyVector().vectorFactory(indices()); } + @Override @ForceInline + public int laneSource(int i) { + return (int)toBitsVector().lane(i); + } + @Override - public FloatMaxShuffle rearrange(VectorShuffle shuffle) { - FloatMaxShuffle s = (FloatMaxShuffle) shuffle; - byte[] reorder1 = reorder(); - byte[] reorder2 = s.reorder(); - byte[] r = new byte[reorder1.length]; - for (int i = 0; i < reorder1.length; i++) { - int ssi = reorder2[i]; - r[i] = reorder1[ssi]; // throws on exceptional index + @ForceInline + public void intoArray(int[] a, int offset) { + toBitsVector().intoArray(a, offset); + } + + private static int[] prepare(int[] indices, int offset) { + int[] a = new int[VLENGTH]; + for (int i = 0; i < VLENGTH; i++) { + int si = indices[offset + i]; + si = partiallyWrapIndex(si, VLENGTH); + a[i] = (int)si; + } + return a; + } + + private static int[] prepare(IntUnaryOperator f) { + int[] a = new int[VLENGTH]; + for (int i = 0; i < VLENGTH; i++) { + int si = f.applyAsInt(i); + si = partiallyWrapIndex(si, VLENGTH); + a[i] = (int)si; + } + return a; + } + + private static boolean indicesInRange(int[] indices) { + int length = indices.length; + for (int si : indices) { + if (si >= (int)length || si < (int)(-length)) { + boolean assertsEnabled = false; + assert(assertsEnabled = true); + if (assertsEnabled) { + String msg = ("index "+si+"out of range ["+length+"] in "+ + java.util.Arrays.toString(indices)); + throw new AssertionError(msg); + } + return false; + } } - return new FloatMaxShuffle(r); + return true; } } diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/FloatVector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/FloatVector.java index 30ed43e302977..e86523870ad72 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/FloatVector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/FloatVector.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, 2022, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2017, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -953,7 +953,7 @@ FloatVector lanewise(VectorOperators.Binary op, // and broadcast, but it would be more surprising not to continue // the obvious pattern started by unary and binary. - /** + /** * {@inheritDoc} * @see #lanewise(VectorOperators.Ternary,float,float,VectorMask) * @see #lanewise(VectorOperators.Ternary,Vector,float,VectorMask) @@ -2334,8 +2334,8 @@ FloatVector rearrangeTemplate(Class shuffletype, } @ForceInline - private final - VectorShuffle toShuffle0(FloatSpecies dsp) { + final + VectorShuffle toShuffle0(AbstractSpecies dsp) { float[] a = toArray(); int[] sa = new int[a.length]; for (int i = 0; i < a.length; i++) { @@ -2344,18 +2344,6 @@ VectorShuffle toShuffle0(FloatSpecies dsp) { return VectorShuffle.fromArray(dsp, sa, 0); } - /*package-private*/ - @ForceInline - final - VectorShuffle toShuffleTemplate(Class shuffleType) { - FloatSpecies vsp = vspecies(); - return VectorSupport.convert(VectorSupport.VECTOR_OP_CAST, - getClass(), float.class, length(), - shuffleType, byte.class, length(), - this, vsp, - FloatVector::toShuffle0); - } - /** * {@inheritDoc} * @since 19 @@ -3649,9 +3637,10 @@ static final class FloatSpecies extends AbstractSpecies { private FloatSpecies(VectorShape shape, Class vectorType, Class> maskType, + Class> shuffleType, Function vectorFactory) { super(shape, LaneType.of(float.class), - vectorType, maskType, + vectorType, maskType, shuffleType, vectorFactory); assert(this.elementSize() == Float.SIZE); } @@ -3927,6 +3916,7 @@ static FloatSpecies species(VectorShape s) { = new FloatSpecies(VectorShape.S_64_BIT, Float64Vector.class, Float64Vector.Float64Mask.class, + Float64Vector.Float64Shuffle.class, Float64Vector::new); /** Species representing {@link FloatVector}s of {@link VectorShape#S_128_BIT VectorShape.S_128_BIT}. */ @@ -3934,6 +3924,7 @@ static FloatSpecies species(VectorShape s) { = new FloatSpecies(VectorShape.S_128_BIT, Float128Vector.class, Float128Vector.Float128Mask.class, + Float128Vector.Float128Shuffle.class, Float128Vector::new); /** Species representing {@link FloatVector}s of {@link VectorShape#S_256_BIT VectorShape.S_256_BIT}. */ @@ -3941,6 +3932,7 @@ static FloatSpecies species(VectorShape s) { = new FloatSpecies(VectorShape.S_256_BIT, Float256Vector.class, Float256Vector.Float256Mask.class, + Float256Vector.Float256Shuffle.class, Float256Vector::new); /** Species representing {@link FloatVector}s of {@link VectorShape#S_512_BIT VectorShape.S_512_BIT}. */ @@ -3948,6 +3940,7 @@ static FloatSpecies species(VectorShape s) { = new FloatSpecies(VectorShape.S_512_BIT, Float512Vector.class, Float512Vector.Float512Mask.class, + Float512Vector.Float512Shuffle.class, Float512Vector::new); /** Species representing {@link FloatVector}s of {@link VectorShape#S_Max_BIT VectorShape.S_Max_BIT}. */ @@ -3955,6 +3948,7 @@ static FloatSpecies species(VectorShape s) { = new FloatSpecies(VectorShape.S_Max_BIT, FloatMaxVector.class, FloatMaxVector.FloatMaxMask.class, + FloatMaxVector.FloatMaxShuffle.class, FloatMaxVector::new); /** diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int128Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int128Vector.java index 158686576f514..d0d8b1817b808 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int128Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int128Vector.java @@ -141,24 +141,9 @@ Int128Mask maskFromArray(boolean[] bits) { @ForceInline Int128Shuffle iotaShuffle() { return Int128Shuffle.IOTA; } - @ForceInline - Int128Shuffle iotaShuffle(int start, int step, boolean wrap) { - if (wrap) { - return (Int128Shuffle)VectorSupport.shuffleIota(ETYPE, Int128Shuffle.class, VSPECIES, VLENGTH, start, step, 1, - (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i*lstep + lstart, l)))); - } else { - return (Int128Shuffle)VectorSupport.shuffleIota(ETYPE, Int128Shuffle.class, VSPECIES, VLENGTH, start, step, 0, - (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (i*lstep + lstart))); - } - } - - @Override - @ForceInline - Int128Shuffle shuffleFromBytes(byte[] reorder) { return new Int128Shuffle(reorder); } - @Override @ForceInline - Int128Shuffle shuffleFromArray(int[] indexes, int i) { return new Int128Shuffle(indexes, i); } + Int128Shuffle shuffleFromArray(int[] indices, int i) { return new Int128Shuffle(indices, i); } @Override @ForceInline @@ -357,9 +342,11 @@ public final long reduceLanesToLong(VectorOperators.Associative op, return (long) super.reduceLanesTemplate(op, Int128Mask.class, (Int128Mask) m); // specialized } + @Override @ForceInline - public VectorShuffle toShuffle() { - return super.toShuffleTemplate(Int128Shuffle.class); // specialize + public final + VectorShuffle toShuffle(AbstractSpecies dsp) { + return super.toShuffleTemplate(dsp); } // Specialized unary testing @@ -788,23 +775,26 @@ static final class Int128Shuffle extends AbstractShuffle { static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM static final Class ETYPE = int.class; // used by the JVM - Int128Shuffle(byte[] reorder) { - super(VLENGTH, reorder); + Int128Shuffle(int[] indices) { + super(indices); + assert(VLENGTH == indices.length); + assert(indicesInRange(indices)); } - public Int128Shuffle(int[] reorder) { - super(VLENGTH, reorder); + Int128Shuffle(int[] indices, int i) { + this(prepare(indices, i)); } - public Int128Shuffle(int[] reorder, int i) { - super(VLENGTH, reorder, i); + Int128Shuffle(IntUnaryOperator fn) { + this(prepare(fn)); } - public Int128Shuffle(IntUnaryOperator fn) { - super(VLENGTH, fn); + int[] indices() { + return (int[])getPayload(); } @Override + @ForceInline public IntSpecies vspecies() { return VSPECIES; } @@ -812,40 +802,70 @@ public IntSpecies vspecies() { static { // There must be enough bits in the shuffle lanes to encode // VLENGTH valid indexes and VLENGTH exceptional ones. - assert(VLENGTH < Byte.MAX_VALUE); - assert(Byte.MIN_VALUE <= -VLENGTH); + assert(VLENGTH < Integer.MAX_VALUE); + assert(Integer.MIN_VALUE <= -VLENGTH); } static final Int128Shuffle IOTA = new Int128Shuffle(IDENTITY); @Override @ForceInline - public Int128Vector toVector() { - return VectorSupport.shuffleToVector(VCLASS, ETYPE, Int128Shuffle.class, this, VLENGTH, - (s) -> ((Int128Vector)(((AbstractShuffle)(s)).toVectorTemplate()))); + Int128Vector toBitsVector() { + return (Int128Vector) super.toBitsVectorTemplate(); } @Override @ForceInline - public VectorShuffle cast(VectorSpecies s) { - AbstractSpecies species = (AbstractSpecies) s; - if (length() != species.laneCount()) - throw new IllegalArgumentException("VectorShuffle length and species length differ"); - int[] shuffleArray = toArray(); - return s.shuffleFromArray(shuffleArray, 0).check(s); + IntVector toBitsVector0() { + return Int128Vector.VSPECIES.dummyVector().vectorFactory(indices()); } + @Override @ForceInline + public int laneSource(int i) { + return (int)toBitsVector().lane(i); + } + @Override - public Int128Shuffle rearrange(VectorShuffle shuffle) { - Int128Shuffle s = (Int128Shuffle) shuffle; - byte[] reorder1 = reorder(); - byte[] reorder2 = s.reorder(); - byte[] r = new byte[reorder1.length]; - for (int i = 0; i < reorder1.length; i++) { - int ssi = reorder2[i]; - r[i] = reorder1[ssi]; // throws on exceptional index + @ForceInline + public void intoArray(int[] a, int offset) { + toBitsVector().intoArray(a, offset); + } + + private static int[] prepare(int[] indices, int offset) { + int[] a = new int[VLENGTH]; + for (int i = 0; i < VLENGTH; i++) { + int si = indices[offset + i]; + si = partiallyWrapIndex(si, VLENGTH); + a[i] = (int)si; + } + return a; + } + + private static int[] prepare(IntUnaryOperator f) { + int[] a = new int[VLENGTH]; + for (int i = 0; i < VLENGTH; i++) { + int si = f.applyAsInt(i); + si = partiallyWrapIndex(si, VLENGTH); + a[i] = (int)si; + } + return a; + } + + private static boolean indicesInRange(int[] indices) { + int length = indices.length; + for (int si : indices) { + if (si >= (int)length || si < (int)(-length)) { + boolean assertsEnabled = false; + assert(assertsEnabled = true); + if (assertsEnabled) { + String msg = ("index "+si+"out of range ["+length+"] in "+ + java.util.Arrays.toString(indices)); + throw new AssertionError(msg); + } + return false; + } } - return new Int128Shuffle(r); + return true; } } diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int256Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int256Vector.java index 4f251e43f1f48..96d4d8569da16 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int256Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int256Vector.java @@ -141,24 +141,9 @@ Int256Mask maskFromArray(boolean[] bits) { @ForceInline Int256Shuffle iotaShuffle() { return Int256Shuffle.IOTA; } - @ForceInline - Int256Shuffle iotaShuffle(int start, int step, boolean wrap) { - if (wrap) { - return (Int256Shuffle)VectorSupport.shuffleIota(ETYPE, Int256Shuffle.class, VSPECIES, VLENGTH, start, step, 1, - (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i*lstep + lstart, l)))); - } else { - return (Int256Shuffle)VectorSupport.shuffleIota(ETYPE, Int256Shuffle.class, VSPECIES, VLENGTH, start, step, 0, - (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (i*lstep + lstart))); - } - } - - @Override - @ForceInline - Int256Shuffle shuffleFromBytes(byte[] reorder) { return new Int256Shuffle(reorder); } - @Override @ForceInline - Int256Shuffle shuffleFromArray(int[] indexes, int i) { return new Int256Shuffle(indexes, i); } + Int256Shuffle shuffleFromArray(int[] indices, int i) { return new Int256Shuffle(indices, i); } @Override @ForceInline @@ -357,9 +342,11 @@ public final long reduceLanesToLong(VectorOperators.Associative op, return (long) super.reduceLanesTemplate(op, Int256Mask.class, (Int256Mask) m); // specialized } + @Override @ForceInline - public VectorShuffle toShuffle() { - return super.toShuffleTemplate(Int256Shuffle.class); // specialize + public final + VectorShuffle toShuffle(AbstractSpecies dsp) { + return super.toShuffleTemplate(dsp); } // Specialized unary testing @@ -796,23 +783,26 @@ static final class Int256Shuffle extends AbstractShuffle { static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM static final Class ETYPE = int.class; // used by the JVM - Int256Shuffle(byte[] reorder) { - super(VLENGTH, reorder); + Int256Shuffle(int[] indices) { + super(indices); + assert(VLENGTH == indices.length); + assert(indicesInRange(indices)); } - public Int256Shuffle(int[] reorder) { - super(VLENGTH, reorder); + Int256Shuffle(int[] indices, int i) { + this(prepare(indices, i)); } - public Int256Shuffle(int[] reorder, int i) { - super(VLENGTH, reorder, i); + Int256Shuffle(IntUnaryOperator fn) { + this(prepare(fn)); } - public Int256Shuffle(IntUnaryOperator fn) { - super(VLENGTH, fn); + int[] indices() { + return (int[])getPayload(); } @Override + @ForceInline public IntSpecies vspecies() { return VSPECIES; } @@ -820,40 +810,70 @@ public IntSpecies vspecies() { static { // There must be enough bits in the shuffle lanes to encode // VLENGTH valid indexes and VLENGTH exceptional ones. - assert(VLENGTH < Byte.MAX_VALUE); - assert(Byte.MIN_VALUE <= -VLENGTH); + assert(VLENGTH < Integer.MAX_VALUE); + assert(Integer.MIN_VALUE <= -VLENGTH); } static final Int256Shuffle IOTA = new Int256Shuffle(IDENTITY); @Override @ForceInline - public Int256Vector toVector() { - return VectorSupport.shuffleToVector(VCLASS, ETYPE, Int256Shuffle.class, this, VLENGTH, - (s) -> ((Int256Vector)(((AbstractShuffle)(s)).toVectorTemplate()))); + Int256Vector toBitsVector() { + return (Int256Vector) super.toBitsVectorTemplate(); } @Override @ForceInline - public VectorShuffle cast(VectorSpecies s) { - AbstractSpecies species = (AbstractSpecies) s; - if (length() != species.laneCount()) - throw new IllegalArgumentException("VectorShuffle length and species length differ"); - int[] shuffleArray = toArray(); - return s.shuffleFromArray(shuffleArray, 0).check(s); + IntVector toBitsVector0() { + return Int256Vector.VSPECIES.dummyVector().vectorFactory(indices()); } + @Override @ForceInline + public int laneSource(int i) { + return (int)toBitsVector().lane(i); + } + @Override - public Int256Shuffle rearrange(VectorShuffle shuffle) { - Int256Shuffle s = (Int256Shuffle) shuffle; - byte[] reorder1 = reorder(); - byte[] reorder2 = s.reorder(); - byte[] r = new byte[reorder1.length]; - for (int i = 0; i < reorder1.length; i++) { - int ssi = reorder2[i]; - r[i] = reorder1[ssi]; // throws on exceptional index + @ForceInline + public void intoArray(int[] a, int offset) { + toBitsVector().intoArray(a, offset); + } + + private static int[] prepare(int[] indices, int offset) { + int[] a = new int[VLENGTH]; + for (int i = 0; i < VLENGTH; i++) { + int si = indices[offset + i]; + si = partiallyWrapIndex(si, VLENGTH); + a[i] = (int)si; + } + return a; + } + + private static int[] prepare(IntUnaryOperator f) { + int[] a = new int[VLENGTH]; + for (int i = 0; i < VLENGTH; i++) { + int si = f.applyAsInt(i); + si = partiallyWrapIndex(si, VLENGTH); + a[i] = (int)si; + } + return a; + } + + private static boolean indicesInRange(int[] indices) { + int length = indices.length; + for (int si : indices) { + if (si >= (int)length || si < (int)(-length)) { + boolean assertsEnabled = false; + assert(assertsEnabled = true); + if (assertsEnabled) { + String msg = ("index "+si+"out of range ["+length+"] in "+ + java.util.Arrays.toString(indices)); + throw new AssertionError(msg); + } + return false; + } } - return new Int256Shuffle(r); + return true; } } diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int512Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int512Vector.java index 4867c377a8a97..7a5b7a63c5289 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int512Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int512Vector.java @@ -141,24 +141,9 @@ Int512Mask maskFromArray(boolean[] bits) { @ForceInline Int512Shuffle iotaShuffle() { return Int512Shuffle.IOTA; } - @ForceInline - Int512Shuffle iotaShuffle(int start, int step, boolean wrap) { - if (wrap) { - return (Int512Shuffle)VectorSupport.shuffleIota(ETYPE, Int512Shuffle.class, VSPECIES, VLENGTH, start, step, 1, - (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i*lstep + lstart, l)))); - } else { - return (Int512Shuffle)VectorSupport.shuffleIota(ETYPE, Int512Shuffle.class, VSPECIES, VLENGTH, start, step, 0, - (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (i*lstep + lstart))); - } - } - - @Override - @ForceInline - Int512Shuffle shuffleFromBytes(byte[] reorder) { return new Int512Shuffle(reorder); } - @Override @ForceInline - Int512Shuffle shuffleFromArray(int[] indexes, int i) { return new Int512Shuffle(indexes, i); } + Int512Shuffle shuffleFromArray(int[] indices, int i) { return new Int512Shuffle(indices, i); } @Override @ForceInline @@ -357,9 +342,11 @@ public final long reduceLanesToLong(VectorOperators.Associative op, return (long) super.reduceLanesTemplate(op, Int512Mask.class, (Int512Mask) m); // specialized } + @Override @ForceInline - public VectorShuffle toShuffle() { - return super.toShuffleTemplate(Int512Shuffle.class); // specialize + public final + VectorShuffle toShuffle(AbstractSpecies dsp) { + return super.toShuffleTemplate(dsp); } // Specialized unary testing @@ -812,23 +799,26 @@ static final class Int512Shuffle extends AbstractShuffle { static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM static final Class ETYPE = int.class; // used by the JVM - Int512Shuffle(byte[] reorder) { - super(VLENGTH, reorder); + Int512Shuffle(int[] indices) { + super(indices); + assert(VLENGTH == indices.length); + assert(indicesInRange(indices)); } - public Int512Shuffle(int[] reorder) { - super(VLENGTH, reorder); + Int512Shuffle(int[] indices, int i) { + this(prepare(indices, i)); } - public Int512Shuffle(int[] reorder, int i) { - super(VLENGTH, reorder, i); + Int512Shuffle(IntUnaryOperator fn) { + this(prepare(fn)); } - public Int512Shuffle(IntUnaryOperator fn) { - super(VLENGTH, fn); + int[] indices() { + return (int[])getPayload(); } @Override + @ForceInline public IntSpecies vspecies() { return VSPECIES; } @@ -836,40 +826,70 @@ public IntSpecies vspecies() { static { // There must be enough bits in the shuffle lanes to encode // VLENGTH valid indexes and VLENGTH exceptional ones. - assert(VLENGTH < Byte.MAX_VALUE); - assert(Byte.MIN_VALUE <= -VLENGTH); + assert(VLENGTH < Integer.MAX_VALUE); + assert(Integer.MIN_VALUE <= -VLENGTH); } static final Int512Shuffle IOTA = new Int512Shuffle(IDENTITY); @Override @ForceInline - public Int512Vector toVector() { - return VectorSupport.shuffleToVector(VCLASS, ETYPE, Int512Shuffle.class, this, VLENGTH, - (s) -> ((Int512Vector)(((AbstractShuffle)(s)).toVectorTemplate()))); + Int512Vector toBitsVector() { + return (Int512Vector) super.toBitsVectorTemplate(); } @Override @ForceInline - public VectorShuffle cast(VectorSpecies s) { - AbstractSpecies species = (AbstractSpecies) s; - if (length() != species.laneCount()) - throw new IllegalArgumentException("VectorShuffle length and species length differ"); - int[] shuffleArray = toArray(); - return s.shuffleFromArray(shuffleArray, 0).check(s); + IntVector toBitsVector0() { + return Int512Vector.VSPECIES.dummyVector().vectorFactory(indices()); } + @Override @ForceInline + public int laneSource(int i) { + return (int)toBitsVector().lane(i); + } + @Override - public Int512Shuffle rearrange(VectorShuffle shuffle) { - Int512Shuffle s = (Int512Shuffle) shuffle; - byte[] reorder1 = reorder(); - byte[] reorder2 = s.reorder(); - byte[] r = new byte[reorder1.length]; - for (int i = 0; i < reorder1.length; i++) { - int ssi = reorder2[i]; - r[i] = reorder1[ssi]; // throws on exceptional index + @ForceInline + public void intoArray(int[] a, int offset) { + toBitsVector().intoArray(a, offset); + } + + private static int[] prepare(int[] indices, int offset) { + int[] a = new int[VLENGTH]; + for (int i = 0; i < VLENGTH; i++) { + int si = indices[offset + i]; + si = partiallyWrapIndex(si, VLENGTH); + a[i] = (int)si; + } + return a; + } + + private static int[] prepare(IntUnaryOperator f) { + int[] a = new int[VLENGTH]; + for (int i = 0; i < VLENGTH; i++) { + int si = f.applyAsInt(i); + si = partiallyWrapIndex(si, VLENGTH); + a[i] = (int)si; + } + return a; + } + + private static boolean indicesInRange(int[] indices) { + int length = indices.length; + for (int si : indices) { + if (si >= (int)length || si < (int)(-length)) { + boolean assertsEnabled = false; + assert(assertsEnabled = true); + if (assertsEnabled) { + String msg = ("index "+si+"out of range ["+length+"] in "+ + java.util.Arrays.toString(indices)); + throw new AssertionError(msg); + } + return false; + } } - return new Int512Shuffle(r); + return true; } } diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int64Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int64Vector.java index 2f44bfecf8f61..d582b0fe0b1c4 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int64Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int64Vector.java @@ -141,24 +141,9 @@ Int64Mask maskFromArray(boolean[] bits) { @ForceInline Int64Shuffle iotaShuffle() { return Int64Shuffle.IOTA; } - @ForceInline - Int64Shuffle iotaShuffle(int start, int step, boolean wrap) { - if (wrap) { - return (Int64Shuffle)VectorSupport.shuffleIota(ETYPE, Int64Shuffle.class, VSPECIES, VLENGTH, start, step, 1, - (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i*lstep + lstart, l)))); - } else { - return (Int64Shuffle)VectorSupport.shuffleIota(ETYPE, Int64Shuffle.class, VSPECIES, VLENGTH, start, step, 0, - (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (i*lstep + lstart))); - } - } - - @Override - @ForceInline - Int64Shuffle shuffleFromBytes(byte[] reorder) { return new Int64Shuffle(reorder); } - @Override @ForceInline - Int64Shuffle shuffleFromArray(int[] indexes, int i) { return new Int64Shuffle(indexes, i); } + Int64Shuffle shuffleFromArray(int[] indices, int i) { return new Int64Shuffle(indices, i); } @Override @ForceInline @@ -357,9 +342,11 @@ public final long reduceLanesToLong(VectorOperators.Associative op, return (long) super.reduceLanesTemplate(op, Int64Mask.class, (Int64Mask) m); // specialized } + @Override @ForceInline - public VectorShuffle toShuffle() { - return super.toShuffleTemplate(Int64Shuffle.class); // specialize + public final + VectorShuffle toShuffle(AbstractSpecies dsp) { + return super.toShuffleTemplate(dsp); } // Specialized unary testing @@ -784,23 +771,26 @@ static final class Int64Shuffle extends AbstractShuffle { static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM static final Class ETYPE = int.class; // used by the JVM - Int64Shuffle(byte[] reorder) { - super(VLENGTH, reorder); + Int64Shuffle(int[] indices) { + super(indices); + assert(VLENGTH == indices.length); + assert(indicesInRange(indices)); } - public Int64Shuffle(int[] reorder) { - super(VLENGTH, reorder); + Int64Shuffle(int[] indices, int i) { + this(prepare(indices, i)); } - public Int64Shuffle(int[] reorder, int i) { - super(VLENGTH, reorder, i); + Int64Shuffle(IntUnaryOperator fn) { + this(prepare(fn)); } - public Int64Shuffle(IntUnaryOperator fn) { - super(VLENGTH, fn); + int[] indices() { + return (int[])getPayload(); } @Override + @ForceInline public IntSpecies vspecies() { return VSPECIES; } @@ -808,40 +798,70 @@ public IntSpecies vspecies() { static { // There must be enough bits in the shuffle lanes to encode // VLENGTH valid indexes and VLENGTH exceptional ones. - assert(VLENGTH < Byte.MAX_VALUE); - assert(Byte.MIN_VALUE <= -VLENGTH); + assert(VLENGTH < Integer.MAX_VALUE); + assert(Integer.MIN_VALUE <= -VLENGTH); } static final Int64Shuffle IOTA = new Int64Shuffle(IDENTITY); @Override @ForceInline - public Int64Vector toVector() { - return VectorSupport.shuffleToVector(VCLASS, ETYPE, Int64Shuffle.class, this, VLENGTH, - (s) -> ((Int64Vector)(((AbstractShuffle)(s)).toVectorTemplate()))); + Int64Vector toBitsVector() { + return (Int64Vector) super.toBitsVectorTemplate(); } @Override @ForceInline - public VectorShuffle cast(VectorSpecies s) { - AbstractSpecies species = (AbstractSpecies) s; - if (length() != species.laneCount()) - throw new IllegalArgumentException("VectorShuffle length and species length differ"); - int[] shuffleArray = toArray(); - return s.shuffleFromArray(shuffleArray, 0).check(s); + IntVector toBitsVector0() { + return Int64Vector.VSPECIES.dummyVector().vectorFactory(indices()); } + @Override @ForceInline + public int laneSource(int i) { + return (int)toBitsVector().lane(i); + } + @Override - public Int64Shuffle rearrange(VectorShuffle shuffle) { - Int64Shuffle s = (Int64Shuffle) shuffle; - byte[] reorder1 = reorder(); - byte[] reorder2 = s.reorder(); - byte[] r = new byte[reorder1.length]; - for (int i = 0; i < reorder1.length; i++) { - int ssi = reorder2[i]; - r[i] = reorder1[ssi]; // throws on exceptional index + @ForceInline + public void intoArray(int[] a, int offset) { + toBitsVector().intoArray(a, offset); + } + + private static int[] prepare(int[] indices, int offset) { + int[] a = new int[VLENGTH]; + for (int i = 0; i < VLENGTH; i++) { + int si = indices[offset + i]; + si = partiallyWrapIndex(si, VLENGTH); + a[i] = (int)si; + } + return a; + } + + private static int[] prepare(IntUnaryOperator f) { + int[] a = new int[VLENGTH]; + for (int i = 0; i < VLENGTH; i++) { + int si = f.applyAsInt(i); + si = partiallyWrapIndex(si, VLENGTH); + a[i] = (int)si; + } + return a; + } + + private static boolean indicesInRange(int[] indices) { + int length = indices.length; + for (int si : indices) { + if (si >= (int)length || si < (int)(-length)) { + boolean assertsEnabled = false; + assert(assertsEnabled = true); + if (assertsEnabled) { + String msg = ("index "+si+"out of range ["+length+"] in "+ + java.util.Arrays.toString(indices)); + throw new AssertionError(msg); + } + return false; + } } - return new Int64Shuffle(r); + return true; } } diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/IntMaxVector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/IntMaxVector.java index b79e73f52d6d1..77c0d07fadb5a 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/IntMaxVector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/IntMaxVector.java @@ -141,24 +141,9 @@ IntMaxMask maskFromArray(boolean[] bits) { @ForceInline IntMaxShuffle iotaShuffle() { return IntMaxShuffle.IOTA; } - @ForceInline - IntMaxShuffle iotaShuffle(int start, int step, boolean wrap) { - if (wrap) { - return (IntMaxShuffle)VectorSupport.shuffleIota(ETYPE, IntMaxShuffle.class, VSPECIES, VLENGTH, start, step, 1, - (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i*lstep + lstart, l)))); - } else { - return (IntMaxShuffle)VectorSupport.shuffleIota(ETYPE, IntMaxShuffle.class, VSPECIES, VLENGTH, start, step, 0, - (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (i*lstep + lstart))); - } - } - - @Override - @ForceInline - IntMaxShuffle shuffleFromBytes(byte[] reorder) { return new IntMaxShuffle(reorder); } - @Override @ForceInline - IntMaxShuffle shuffleFromArray(int[] indexes, int i) { return new IntMaxShuffle(indexes, i); } + IntMaxShuffle shuffleFromArray(int[] indices, int i) { return new IntMaxShuffle(indices, i); } @Override @ForceInline @@ -357,9 +342,11 @@ public final long reduceLanesToLong(VectorOperators.Associative op, return (long) super.reduceLanesTemplate(op, IntMaxMask.class, (IntMaxMask) m); // specialized } + @Override @ForceInline - public VectorShuffle toShuffle() { - return super.toShuffleTemplate(IntMaxShuffle.class); // specialize + public final + VectorShuffle toShuffle(AbstractSpecies dsp) { + return super.toShuffleTemplate(dsp); } // Specialized unary testing @@ -793,23 +780,26 @@ static final class IntMaxShuffle extends AbstractShuffle { static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM static final Class ETYPE = int.class; // used by the JVM - IntMaxShuffle(byte[] reorder) { - super(VLENGTH, reorder); + IntMaxShuffle(int[] indices) { + super(indices); + assert(VLENGTH == indices.length); + assert(indicesInRange(indices)); } - public IntMaxShuffle(int[] reorder) { - super(VLENGTH, reorder); + IntMaxShuffle(int[] indices, int i) { + this(prepare(indices, i)); } - public IntMaxShuffle(int[] reorder, int i) { - super(VLENGTH, reorder, i); + IntMaxShuffle(IntUnaryOperator fn) { + this(prepare(fn)); } - public IntMaxShuffle(IntUnaryOperator fn) { - super(VLENGTH, fn); + int[] indices() { + return (int[])getPayload(); } @Override + @ForceInline public IntSpecies vspecies() { return VSPECIES; } @@ -817,40 +807,70 @@ public IntSpecies vspecies() { static { // There must be enough bits in the shuffle lanes to encode // VLENGTH valid indexes and VLENGTH exceptional ones. - assert(VLENGTH < Byte.MAX_VALUE); - assert(Byte.MIN_VALUE <= -VLENGTH); + assert(VLENGTH < Integer.MAX_VALUE); + assert(Integer.MIN_VALUE <= -VLENGTH); } static final IntMaxShuffle IOTA = new IntMaxShuffle(IDENTITY); @Override @ForceInline - public IntMaxVector toVector() { - return VectorSupport.shuffleToVector(VCLASS, ETYPE, IntMaxShuffle.class, this, VLENGTH, - (s) -> ((IntMaxVector)(((AbstractShuffle)(s)).toVectorTemplate()))); + IntMaxVector toBitsVector() { + return (IntMaxVector) super.toBitsVectorTemplate(); } @Override @ForceInline - public VectorShuffle cast(VectorSpecies s) { - AbstractSpecies species = (AbstractSpecies) s; - if (length() != species.laneCount()) - throw new IllegalArgumentException("VectorShuffle length and species length differ"); - int[] shuffleArray = toArray(); - return s.shuffleFromArray(shuffleArray, 0).check(s); + IntVector toBitsVector0() { + return IntMaxVector.VSPECIES.dummyVector().vectorFactory(indices()); } + @Override @ForceInline + public int laneSource(int i) { + return (int)toBitsVector().lane(i); + } + @Override - public IntMaxShuffle rearrange(VectorShuffle shuffle) { - IntMaxShuffle s = (IntMaxShuffle) shuffle; - byte[] reorder1 = reorder(); - byte[] reorder2 = s.reorder(); - byte[] r = new byte[reorder1.length]; - for (int i = 0; i < reorder1.length; i++) { - int ssi = reorder2[i]; - r[i] = reorder1[ssi]; // throws on exceptional index + @ForceInline + public void intoArray(int[] a, int offset) { + toBitsVector().intoArray(a, offset); + } + + private static int[] prepare(int[] indices, int offset) { + int[] a = new int[VLENGTH]; + for (int i = 0; i < VLENGTH; i++) { + int si = indices[offset + i]; + si = partiallyWrapIndex(si, VLENGTH); + a[i] = (int)si; + } + return a; + } + + private static int[] prepare(IntUnaryOperator f) { + int[] a = new int[VLENGTH]; + for (int i = 0; i < VLENGTH; i++) { + int si = f.applyAsInt(i); + si = partiallyWrapIndex(si, VLENGTH); + a[i] = (int)si; + } + return a; + } + + private static boolean indicesInRange(int[] indices) { + int length = indices.length; + for (int si : indices) { + if (si >= (int)length || si < (int)(-length)) { + boolean assertsEnabled = false; + assert(assertsEnabled = true); + if (assertsEnabled) { + String msg = ("index "+si+"out of range ["+length+"] in "+ + java.util.Arrays.toString(indices)); + throw new AssertionError(msg); + } + return false; + } } - return new IntMaxShuffle(r); + return true; } } diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/IntVector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/IntVector.java index 87787b9a61199..a6faa907387d0 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/IntVector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/IntVector.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, 2022, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2017, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -1074,7 +1074,7 @@ private static VectorBroadcastIntOp> broadcastInt // and broadcast, but it would be more surprising not to continue // the obvious pattern started by unary and binary. - /** + /** * {@inheritDoc} * @see #lanewise(VectorOperators.Ternary,int,int,VectorMask) * @see #lanewise(VectorOperators.Ternary,Vector,int,VectorMask) @@ -2465,8 +2465,8 @@ IntVector rearrangeTemplate(Class shuffletype, } @ForceInline - private final - VectorShuffle toShuffle0(IntSpecies dsp) { + final + VectorShuffle toShuffle0(AbstractSpecies dsp) { int[] a = toArray(); int[] sa = new int[a.length]; for (int i = 0; i < a.length; i++) { @@ -2475,18 +2475,6 @@ VectorShuffle toShuffle0(IntSpecies dsp) { return VectorShuffle.fromArray(dsp, sa, 0); } - /*package-private*/ - @ForceInline - final - VectorShuffle toShuffleTemplate(Class shuffleType) { - IntSpecies vsp = vspecies(); - return VectorSupport.convert(VectorSupport.VECTOR_OP_CAST, - getClass(), int.class, length(), - shuffleType, byte.class, length(), - this, vsp, - IntVector::toShuffle0); - } - /** * {@inheritDoc} * @since 19 @@ -3805,9 +3793,10 @@ static final class IntSpecies extends AbstractSpecies { private IntSpecies(VectorShape shape, Class vectorType, Class> maskType, + Class> shuffleType, Function vectorFactory) { super(shape, LaneType.of(int.class), - vectorType, maskType, + vectorType, maskType, shuffleType, vectorFactory); assert(this.elementSize() == Integer.SIZE); } @@ -4083,6 +4072,7 @@ static IntSpecies species(VectorShape s) { = new IntSpecies(VectorShape.S_64_BIT, Int64Vector.class, Int64Vector.Int64Mask.class, + Int64Vector.Int64Shuffle.class, Int64Vector::new); /** Species representing {@link IntVector}s of {@link VectorShape#S_128_BIT VectorShape.S_128_BIT}. */ @@ -4090,6 +4080,7 @@ static IntSpecies species(VectorShape s) { = new IntSpecies(VectorShape.S_128_BIT, Int128Vector.class, Int128Vector.Int128Mask.class, + Int128Vector.Int128Shuffle.class, Int128Vector::new); /** Species representing {@link IntVector}s of {@link VectorShape#S_256_BIT VectorShape.S_256_BIT}. */ @@ -4097,6 +4088,7 @@ static IntSpecies species(VectorShape s) { = new IntSpecies(VectorShape.S_256_BIT, Int256Vector.class, Int256Vector.Int256Mask.class, + Int256Vector.Int256Shuffle.class, Int256Vector::new); /** Species representing {@link IntVector}s of {@link VectorShape#S_512_BIT VectorShape.S_512_BIT}. */ @@ -4104,6 +4096,7 @@ static IntSpecies species(VectorShape s) { = new IntSpecies(VectorShape.S_512_BIT, Int512Vector.class, Int512Vector.Int512Mask.class, + Int512Vector.Int512Shuffle.class, Int512Vector::new); /** Species representing {@link IntVector}s of {@link VectorShape#S_Max_BIT VectorShape.S_Max_BIT}. */ @@ -4111,6 +4104,7 @@ static IntSpecies species(VectorShape s) { = new IntSpecies(VectorShape.S_Max_BIT, IntMaxVector.class, IntMaxVector.IntMaxMask.class, + IntMaxVector.IntMaxShuffle.class, IntMaxVector::new); /** diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long128Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long128Vector.java index 9b024f6e55cba..f9fa51b220cd4 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long128Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long128Vector.java @@ -136,24 +136,9 @@ Long128Mask maskFromArray(boolean[] bits) { @ForceInline Long128Shuffle iotaShuffle() { return Long128Shuffle.IOTA; } - @ForceInline - Long128Shuffle iotaShuffle(int start, int step, boolean wrap) { - if (wrap) { - return (Long128Shuffle)VectorSupport.shuffleIota(ETYPE, Long128Shuffle.class, VSPECIES, VLENGTH, start, step, 1, - (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i*lstep + lstart, l)))); - } else { - return (Long128Shuffle)VectorSupport.shuffleIota(ETYPE, Long128Shuffle.class, VSPECIES, VLENGTH, start, step, 0, - (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (i*lstep + lstart))); - } - } - @Override @ForceInline - Long128Shuffle shuffleFromBytes(byte[] reorder) { return new Long128Shuffle(reorder); } - - @Override - @ForceInline - Long128Shuffle shuffleFromArray(int[] indexes, int i) { return new Long128Shuffle(indexes, i); } + Long128Shuffle shuffleFromArray(int[] indices, int i) { return new Long128Shuffle(indices, i); } @Override @ForceInline @@ -352,9 +337,11 @@ public final long reduceLanesToLong(VectorOperators.Associative op, return (long) super.reduceLanesTemplate(op, Long128Mask.class, (Long128Mask) m); // specialized } + @Override @ForceInline - public VectorShuffle toShuffle() { - return super.toShuffleTemplate(Long128Shuffle.class); // specialize + public final + VectorShuffle toShuffle(AbstractSpecies dsp) { + return super.toShuffleTemplate(dsp); } // Specialized unary testing @@ -774,23 +761,26 @@ static final class Long128Shuffle extends AbstractShuffle { static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM static final Class ETYPE = long.class; // used by the JVM - Long128Shuffle(byte[] reorder) { - super(VLENGTH, reorder); + Long128Shuffle(long[] indices) { + super(indices); + assert(VLENGTH == indices.length); + assert(indicesInRange(indices)); } - public Long128Shuffle(int[] reorder) { - super(VLENGTH, reorder); + Long128Shuffle(int[] indices, int i) { + this(prepare(indices, i)); } - public Long128Shuffle(int[] reorder, int i) { - super(VLENGTH, reorder, i); + Long128Shuffle(IntUnaryOperator fn) { + this(prepare(fn)); } - public Long128Shuffle(IntUnaryOperator fn) { - super(VLENGTH, fn); + long[] indices() { + return (long[])getPayload(); } @Override + @ForceInline public LongSpecies vspecies() { return VSPECIES; } @@ -798,40 +788,76 @@ public LongSpecies vspecies() { static { // There must be enough bits in the shuffle lanes to encode // VLENGTH valid indexes and VLENGTH exceptional ones. - assert(VLENGTH < Byte.MAX_VALUE); - assert(Byte.MIN_VALUE <= -VLENGTH); + assert(VLENGTH < Long.MAX_VALUE); + assert(Long.MIN_VALUE <= -VLENGTH); } static final Long128Shuffle IOTA = new Long128Shuffle(IDENTITY); @Override @ForceInline - public Long128Vector toVector() { - return VectorSupport.shuffleToVector(VCLASS, ETYPE, Long128Shuffle.class, this, VLENGTH, - (s) -> ((Long128Vector)(((AbstractShuffle)(s)).toVectorTemplate()))); + Long128Vector toBitsVector() { + return (Long128Vector) super.toBitsVectorTemplate(); } @Override @ForceInline - public VectorShuffle cast(VectorSpecies s) { - AbstractSpecies species = (AbstractSpecies) s; - if (length() != species.laneCount()) - throw new IllegalArgumentException("VectorShuffle length and species length differ"); - int[] shuffleArray = toArray(); - return s.shuffleFromArray(shuffleArray, 0).check(s); + LongVector toBitsVector0() { + return Long128Vector.VSPECIES.dummyVector().vectorFactory(indices()); } + @Override @ForceInline + public int laneSource(int i) { + return (int)toBitsVector().lane(i); + } + @Override - public Long128Shuffle rearrange(VectorShuffle shuffle) { - Long128Shuffle s = (Long128Shuffle) shuffle; - byte[] reorder1 = reorder(); - byte[] reorder2 = s.reorder(); - byte[] r = new byte[reorder1.length]; - for (int i = 0; i < reorder1.length; i++) { - int ssi = reorder2[i]; - r[i] = reorder1[ssi]; // throws on exceptional index + @ForceInline + public void intoArray(int[] a, int offset) { + VectorSpecies species = VectorSpecies.of( + int.class, + VectorShape.forBitSize(length() * Integer.SIZE)); + Vector v = toBitsVector(); + v.convertShape(VectorOperators.L2I, species, 0) + .reinterpretAsInts() + .intoArray(a, offset); + } + + private static long[] prepare(int[] indices, int offset) { + long[] a = new long[VLENGTH]; + for (int i = 0; i < VLENGTH; i++) { + int si = indices[offset + i]; + si = partiallyWrapIndex(si, VLENGTH); + a[i] = (long)si; + } + return a; + } + + private static long[] prepare(IntUnaryOperator f) { + long[] a = new long[VLENGTH]; + for (int i = 0; i < VLENGTH; i++) { + int si = f.applyAsInt(i); + si = partiallyWrapIndex(si, VLENGTH); + a[i] = (long)si; + } + return a; + } + + private static boolean indicesInRange(long[] indices) { + int length = indices.length; + for (long si : indices) { + if (si >= (long)length || si < (long)(-length)) { + boolean assertsEnabled = false; + assert(assertsEnabled = true); + if (assertsEnabled) { + String msg = ("index "+si+"out of range ["+length+"] in "+ + java.util.Arrays.toString(indices)); + throw new AssertionError(msg); + } + return false; + } } - return new Long128Shuffle(r); + return true; } } diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long256Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long256Vector.java index 8a1844af3702b..9c666de8bd88e 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long256Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long256Vector.java @@ -136,24 +136,9 @@ Long256Mask maskFromArray(boolean[] bits) { @ForceInline Long256Shuffle iotaShuffle() { return Long256Shuffle.IOTA; } - @ForceInline - Long256Shuffle iotaShuffle(int start, int step, boolean wrap) { - if (wrap) { - return (Long256Shuffle)VectorSupport.shuffleIota(ETYPE, Long256Shuffle.class, VSPECIES, VLENGTH, start, step, 1, - (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i*lstep + lstart, l)))); - } else { - return (Long256Shuffle)VectorSupport.shuffleIota(ETYPE, Long256Shuffle.class, VSPECIES, VLENGTH, start, step, 0, - (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (i*lstep + lstart))); - } - } - @Override @ForceInline - Long256Shuffle shuffleFromBytes(byte[] reorder) { return new Long256Shuffle(reorder); } - - @Override - @ForceInline - Long256Shuffle shuffleFromArray(int[] indexes, int i) { return new Long256Shuffle(indexes, i); } + Long256Shuffle shuffleFromArray(int[] indices, int i) { return new Long256Shuffle(indices, i); } @Override @ForceInline @@ -352,9 +337,11 @@ public final long reduceLanesToLong(VectorOperators.Associative op, return (long) super.reduceLanesTemplate(op, Long256Mask.class, (Long256Mask) m); // specialized } + @Override @ForceInline - public VectorShuffle toShuffle() { - return super.toShuffleTemplate(Long256Shuffle.class); // specialize + public final + VectorShuffle toShuffle(AbstractSpecies dsp) { + return super.toShuffleTemplate(dsp); } // Specialized unary testing @@ -778,23 +765,26 @@ static final class Long256Shuffle extends AbstractShuffle { static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM static final Class ETYPE = long.class; // used by the JVM - Long256Shuffle(byte[] reorder) { - super(VLENGTH, reorder); + Long256Shuffle(long[] indices) { + super(indices); + assert(VLENGTH == indices.length); + assert(indicesInRange(indices)); } - public Long256Shuffle(int[] reorder) { - super(VLENGTH, reorder); + Long256Shuffle(int[] indices, int i) { + this(prepare(indices, i)); } - public Long256Shuffle(int[] reorder, int i) { - super(VLENGTH, reorder, i); + Long256Shuffle(IntUnaryOperator fn) { + this(prepare(fn)); } - public Long256Shuffle(IntUnaryOperator fn) { - super(VLENGTH, fn); + long[] indices() { + return (long[])getPayload(); } @Override + @ForceInline public LongSpecies vspecies() { return VSPECIES; } @@ -802,40 +792,76 @@ public LongSpecies vspecies() { static { // There must be enough bits in the shuffle lanes to encode // VLENGTH valid indexes and VLENGTH exceptional ones. - assert(VLENGTH < Byte.MAX_VALUE); - assert(Byte.MIN_VALUE <= -VLENGTH); + assert(VLENGTH < Long.MAX_VALUE); + assert(Long.MIN_VALUE <= -VLENGTH); } static final Long256Shuffle IOTA = new Long256Shuffle(IDENTITY); @Override @ForceInline - public Long256Vector toVector() { - return VectorSupport.shuffleToVector(VCLASS, ETYPE, Long256Shuffle.class, this, VLENGTH, - (s) -> ((Long256Vector)(((AbstractShuffle)(s)).toVectorTemplate()))); + Long256Vector toBitsVector() { + return (Long256Vector) super.toBitsVectorTemplate(); } @Override @ForceInline - public VectorShuffle cast(VectorSpecies s) { - AbstractSpecies species = (AbstractSpecies) s; - if (length() != species.laneCount()) - throw new IllegalArgumentException("VectorShuffle length and species length differ"); - int[] shuffleArray = toArray(); - return s.shuffleFromArray(shuffleArray, 0).check(s); + LongVector toBitsVector0() { + return Long256Vector.VSPECIES.dummyVector().vectorFactory(indices()); } + @Override @ForceInline + public int laneSource(int i) { + return (int)toBitsVector().lane(i); + } + @Override - public Long256Shuffle rearrange(VectorShuffle shuffle) { - Long256Shuffle s = (Long256Shuffle) shuffle; - byte[] reorder1 = reorder(); - byte[] reorder2 = s.reorder(); - byte[] r = new byte[reorder1.length]; - for (int i = 0; i < reorder1.length; i++) { - int ssi = reorder2[i]; - r[i] = reorder1[ssi]; // throws on exceptional index + @ForceInline + public void intoArray(int[] a, int offset) { + VectorSpecies species = VectorSpecies.of( + int.class, + VectorShape.forBitSize(length() * Integer.SIZE)); + Vector v = toBitsVector(); + v.convertShape(VectorOperators.L2I, species, 0) + .reinterpretAsInts() + .intoArray(a, offset); + } + + private static long[] prepare(int[] indices, int offset) { + long[] a = new long[VLENGTH]; + for (int i = 0; i < VLENGTH; i++) { + int si = indices[offset + i]; + si = partiallyWrapIndex(si, VLENGTH); + a[i] = (long)si; + } + return a; + } + + private static long[] prepare(IntUnaryOperator f) { + long[] a = new long[VLENGTH]; + for (int i = 0; i < VLENGTH; i++) { + int si = f.applyAsInt(i); + si = partiallyWrapIndex(si, VLENGTH); + a[i] = (long)si; + } + return a; + } + + private static boolean indicesInRange(long[] indices) { + int length = indices.length; + for (long si : indices) { + if (si >= (long)length || si < (long)(-length)) { + boolean assertsEnabled = false; + assert(assertsEnabled = true); + if (assertsEnabled) { + String msg = ("index "+si+"out of range ["+length+"] in "+ + java.util.Arrays.toString(indices)); + throw new AssertionError(msg); + } + return false; + } } - return new Long256Shuffle(r); + return true; } } diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long512Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long512Vector.java index baec813f58eb3..edf796bbd2009 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long512Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long512Vector.java @@ -136,24 +136,9 @@ Long512Mask maskFromArray(boolean[] bits) { @ForceInline Long512Shuffle iotaShuffle() { return Long512Shuffle.IOTA; } - @ForceInline - Long512Shuffle iotaShuffle(int start, int step, boolean wrap) { - if (wrap) { - return (Long512Shuffle)VectorSupport.shuffleIota(ETYPE, Long512Shuffle.class, VSPECIES, VLENGTH, start, step, 1, - (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i*lstep + lstart, l)))); - } else { - return (Long512Shuffle)VectorSupport.shuffleIota(ETYPE, Long512Shuffle.class, VSPECIES, VLENGTH, start, step, 0, - (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (i*lstep + lstart))); - } - } - @Override @ForceInline - Long512Shuffle shuffleFromBytes(byte[] reorder) { return new Long512Shuffle(reorder); } - - @Override - @ForceInline - Long512Shuffle shuffleFromArray(int[] indexes, int i) { return new Long512Shuffle(indexes, i); } + Long512Shuffle shuffleFromArray(int[] indices, int i) { return new Long512Shuffle(indices, i); } @Override @ForceInline @@ -352,9 +337,11 @@ public final long reduceLanesToLong(VectorOperators.Associative op, return (long) super.reduceLanesTemplate(op, Long512Mask.class, (Long512Mask) m); // specialized } + @Override @ForceInline - public VectorShuffle toShuffle() { - return super.toShuffleTemplate(Long512Shuffle.class); // specialize + public final + VectorShuffle toShuffle(AbstractSpecies dsp) { + return super.toShuffleTemplate(dsp); } // Specialized unary testing @@ -786,23 +773,26 @@ static final class Long512Shuffle extends AbstractShuffle { static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM static final Class ETYPE = long.class; // used by the JVM - Long512Shuffle(byte[] reorder) { - super(VLENGTH, reorder); + Long512Shuffle(long[] indices) { + super(indices); + assert(VLENGTH == indices.length); + assert(indicesInRange(indices)); } - public Long512Shuffle(int[] reorder) { - super(VLENGTH, reorder); + Long512Shuffle(int[] indices, int i) { + this(prepare(indices, i)); } - public Long512Shuffle(int[] reorder, int i) { - super(VLENGTH, reorder, i); + Long512Shuffle(IntUnaryOperator fn) { + this(prepare(fn)); } - public Long512Shuffle(IntUnaryOperator fn) { - super(VLENGTH, fn); + long[] indices() { + return (long[])getPayload(); } @Override + @ForceInline public LongSpecies vspecies() { return VSPECIES; } @@ -810,40 +800,76 @@ public LongSpecies vspecies() { static { // There must be enough bits in the shuffle lanes to encode // VLENGTH valid indexes and VLENGTH exceptional ones. - assert(VLENGTH < Byte.MAX_VALUE); - assert(Byte.MIN_VALUE <= -VLENGTH); + assert(VLENGTH < Long.MAX_VALUE); + assert(Long.MIN_VALUE <= -VLENGTH); } static final Long512Shuffle IOTA = new Long512Shuffle(IDENTITY); @Override @ForceInline - public Long512Vector toVector() { - return VectorSupport.shuffleToVector(VCLASS, ETYPE, Long512Shuffle.class, this, VLENGTH, - (s) -> ((Long512Vector)(((AbstractShuffle)(s)).toVectorTemplate()))); + Long512Vector toBitsVector() { + return (Long512Vector) super.toBitsVectorTemplate(); } @Override @ForceInline - public VectorShuffle cast(VectorSpecies s) { - AbstractSpecies species = (AbstractSpecies) s; - if (length() != species.laneCount()) - throw new IllegalArgumentException("VectorShuffle length and species length differ"); - int[] shuffleArray = toArray(); - return s.shuffleFromArray(shuffleArray, 0).check(s); + LongVector toBitsVector0() { + return Long512Vector.VSPECIES.dummyVector().vectorFactory(indices()); } + @Override @ForceInline + public int laneSource(int i) { + return (int)toBitsVector().lane(i); + } + @Override - public Long512Shuffle rearrange(VectorShuffle shuffle) { - Long512Shuffle s = (Long512Shuffle) shuffle; - byte[] reorder1 = reorder(); - byte[] reorder2 = s.reorder(); - byte[] r = new byte[reorder1.length]; - for (int i = 0; i < reorder1.length; i++) { - int ssi = reorder2[i]; - r[i] = reorder1[ssi]; // throws on exceptional index + @ForceInline + public void intoArray(int[] a, int offset) { + VectorSpecies species = VectorSpecies.of( + int.class, + VectorShape.forBitSize(length() * Integer.SIZE)); + Vector v = toBitsVector(); + v.convertShape(VectorOperators.L2I, species, 0) + .reinterpretAsInts() + .intoArray(a, offset); + } + + private static long[] prepare(int[] indices, int offset) { + long[] a = new long[VLENGTH]; + for (int i = 0; i < VLENGTH; i++) { + int si = indices[offset + i]; + si = partiallyWrapIndex(si, VLENGTH); + a[i] = (long)si; + } + return a; + } + + private static long[] prepare(IntUnaryOperator f) { + long[] a = new long[VLENGTH]; + for (int i = 0; i < VLENGTH; i++) { + int si = f.applyAsInt(i); + si = partiallyWrapIndex(si, VLENGTH); + a[i] = (long)si; + } + return a; + } + + private static boolean indicesInRange(long[] indices) { + int length = indices.length; + for (long si : indices) { + if (si >= (long)length || si < (long)(-length)) { + boolean assertsEnabled = false; + assert(assertsEnabled = true); + if (assertsEnabled) { + String msg = ("index "+si+"out of range ["+length+"] in "+ + java.util.Arrays.toString(indices)); + throw new AssertionError(msg); + } + return false; + } } - return new Long512Shuffle(r); + return true; } } diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long64Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long64Vector.java index 1d4110050f2d2..a1739f1b1ca88 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long64Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long64Vector.java @@ -136,24 +136,9 @@ Long64Mask maskFromArray(boolean[] bits) { @ForceInline Long64Shuffle iotaShuffle() { return Long64Shuffle.IOTA; } - @ForceInline - Long64Shuffle iotaShuffle(int start, int step, boolean wrap) { - if (wrap) { - return (Long64Shuffle)VectorSupport.shuffleIota(ETYPE, Long64Shuffle.class, VSPECIES, VLENGTH, start, step, 1, - (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i*lstep + lstart, l)))); - } else { - return (Long64Shuffle)VectorSupport.shuffleIota(ETYPE, Long64Shuffle.class, VSPECIES, VLENGTH, start, step, 0, - (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (i*lstep + lstart))); - } - } - - @Override - @ForceInline - Long64Shuffle shuffleFromBytes(byte[] reorder) { return new Long64Shuffle(reorder); } - @Override @ForceInline - Long64Shuffle shuffleFromArray(int[] indexes, int i) { return new Long64Shuffle(indexes, i); } + Long64Shuffle shuffleFromArray(int[] indices, int i) { return new Long64Shuffle(indices, i); } @Override @ForceInline @@ -352,9 +337,11 @@ public final long reduceLanesToLong(VectorOperators.Associative op, return (long) super.reduceLanesTemplate(op, Long64Mask.class, (Long64Mask) m); // specialized } + @Override @ForceInline - public VectorShuffle toShuffle() { - return super.toShuffleTemplate(Long64Shuffle.class); // specialize + public final + VectorShuffle toShuffle(AbstractSpecies dsp) { + return super.toShuffleTemplate(dsp); } // Specialized unary testing @@ -772,23 +759,26 @@ static final class Long64Shuffle extends AbstractShuffle { static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM static final Class ETYPE = long.class; // used by the JVM - Long64Shuffle(byte[] reorder) { - super(VLENGTH, reorder); + Long64Shuffle(long[] indices) { + super(indices); + assert(VLENGTH == indices.length); + assert(indicesInRange(indices)); } - public Long64Shuffle(int[] reorder) { - super(VLENGTH, reorder); + Long64Shuffle(int[] indices, int i) { + this(prepare(indices, i)); } - public Long64Shuffle(int[] reorder, int i) { - super(VLENGTH, reorder, i); + Long64Shuffle(IntUnaryOperator fn) { + this(prepare(fn)); } - public Long64Shuffle(IntUnaryOperator fn) { - super(VLENGTH, fn); + long[] indices() { + return (long[])getPayload(); } @Override + @ForceInline public LongSpecies vspecies() { return VSPECIES; } @@ -796,40 +786,70 @@ public LongSpecies vspecies() { static { // There must be enough bits in the shuffle lanes to encode // VLENGTH valid indexes and VLENGTH exceptional ones. - assert(VLENGTH < Byte.MAX_VALUE); - assert(Byte.MIN_VALUE <= -VLENGTH); + assert(VLENGTH < Long.MAX_VALUE); + assert(Long.MIN_VALUE <= -VLENGTH); } static final Long64Shuffle IOTA = new Long64Shuffle(IDENTITY); @Override @ForceInline - public Long64Vector toVector() { - return VectorSupport.shuffleToVector(VCLASS, ETYPE, Long64Shuffle.class, this, VLENGTH, - (s) -> ((Long64Vector)(((AbstractShuffle)(s)).toVectorTemplate()))); + Long64Vector toBitsVector() { + return (Long64Vector) super.toBitsVectorTemplate(); } @Override @ForceInline - public VectorShuffle cast(VectorSpecies s) { - AbstractSpecies species = (AbstractSpecies) s; - if (length() != species.laneCount()) - throw new IllegalArgumentException("VectorShuffle length and species length differ"); - int[] shuffleArray = toArray(); - return s.shuffleFromArray(shuffleArray, 0).check(s); + LongVector toBitsVector0() { + return Long64Vector.VSPECIES.dummyVector().vectorFactory(indices()); } + @Override @ForceInline + public int laneSource(int i) { + return (int)toBitsVector().lane(i); + } + @Override - public Long64Shuffle rearrange(VectorShuffle shuffle) { - Long64Shuffle s = (Long64Shuffle) shuffle; - byte[] reorder1 = reorder(); - byte[] reorder2 = s.reorder(); - byte[] r = new byte[reorder1.length]; - for (int i = 0; i < reorder1.length; i++) { - int ssi = reorder2[i]; - r[i] = reorder1[ssi]; // throws on exceptional index + @ForceInline + public void intoArray(int[] a, int offset) { + a[offset] = laneSource(0); + } + + private static long[] prepare(int[] indices, int offset) { + long[] a = new long[VLENGTH]; + for (int i = 0; i < VLENGTH; i++) { + int si = indices[offset + i]; + si = partiallyWrapIndex(si, VLENGTH); + a[i] = (long)si; + } + return a; + } + + private static long[] prepare(IntUnaryOperator f) { + long[] a = new long[VLENGTH]; + for (int i = 0; i < VLENGTH; i++) { + int si = f.applyAsInt(i); + si = partiallyWrapIndex(si, VLENGTH); + a[i] = (long)si; + } + return a; + } + + private static boolean indicesInRange(long[] indices) { + int length = indices.length; + for (long si : indices) { + if (si >= (long)length || si < (long)(-length)) { + boolean assertsEnabled = false; + assert(assertsEnabled = true); + if (assertsEnabled) { + String msg = ("index "+si+"out of range ["+length+"] in "+ + java.util.Arrays.toString(indices)); + throw new AssertionError(msg); + } + return false; + } } - return new Long64Shuffle(r); + return true; } } diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/LongMaxVector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/LongMaxVector.java index ecc2b393061d4..d82895278f92f 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/LongMaxVector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/LongMaxVector.java @@ -136,24 +136,9 @@ LongMaxMask maskFromArray(boolean[] bits) { @ForceInline LongMaxShuffle iotaShuffle() { return LongMaxShuffle.IOTA; } - @ForceInline - LongMaxShuffle iotaShuffle(int start, int step, boolean wrap) { - if (wrap) { - return (LongMaxShuffle)VectorSupport.shuffleIota(ETYPE, LongMaxShuffle.class, VSPECIES, VLENGTH, start, step, 1, - (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i*lstep + lstart, l)))); - } else { - return (LongMaxShuffle)VectorSupport.shuffleIota(ETYPE, LongMaxShuffle.class, VSPECIES, VLENGTH, start, step, 0, - (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (i*lstep + lstart))); - } - } - @Override @ForceInline - LongMaxShuffle shuffleFromBytes(byte[] reorder) { return new LongMaxShuffle(reorder); } - - @Override - @ForceInline - LongMaxShuffle shuffleFromArray(int[] indexes, int i) { return new LongMaxShuffle(indexes, i); } + LongMaxShuffle shuffleFromArray(int[] indices, int i) { return new LongMaxShuffle(indices, i); } @Override @ForceInline @@ -352,9 +337,11 @@ public final long reduceLanesToLong(VectorOperators.Associative op, return (long) super.reduceLanesTemplate(op, LongMaxMask.class, (LongMaxMask) m); // specialized } + @Override @ForceInline - public VectorShuffle toShuffle() { - return super.toShuffleTemplate(LongMaxShuffle.class); // specialize + public final + VectorShuffle toShuffle(AbstractSpecies dsp) { + return super.toShuffleTemplate(dsp); } // Specialized unary testing @@ -772,23 +759,26 @@ static final class LongMaxShuffle extends AbstractShuffle { static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM static final Class ETYPE = long.class; // used by the JVM - LongMaxShuffle(byte[] reorder) { - super(VLENGTH, reorder); + LongMaxShuffle(long[] indices) { + super(indices); + assert(VLENGTH == indices.length); + assert(indicesInRange(indices)); } - public LongMaxShuffle(int[] reorder) { - super(VLENGTH, reorder); + LongMaxShuffle(int[] indices, int i) { + this(prepare(indices, i)); } - public LongMaxShuffle(int[] reorder, int i) { - super(VLENGTH, reorder, i); + LongMaxShuffle(IntUnaryOperator fn) { + this(prepare(fn)); } - public LongMaxShuffle(IntUnaryOperator fn) { - super(VLENGTH, fn); + long[] indices() { + return (long[])getPayload(); } @Override + @ForceInline public LongSpecies vspecies() { return VSPECIES; } @@ -796,40 +786,76 @@ public LongSpecies vspecies() { static { // There must be enough bits in the shuffle lanes to encode // VLENGTH valid indexes and VLENGTH exceptional ones. - assert(VLENGTH < Byte.MAX_VALUE); - assert(Byte.MIN_VALUE <= -VLENGTH); + assert(VLENGTH < Long.MAX_VALUE); + assert(Long.MIN_VALUE <= -VLENGTH); } static final LongMaxShuffle IOTA = new LongMaxShuffle(IDENTITY); @Override @ForceInline - public LongMaxVector toVector() { - return VectorSupport.shuffleToVector(VCLASS, ETYPE, LongMaxShuffle.class, this, VLENGTH, - (s) -> ((LongMaxVector)(((AbstractShuffle)(s)).toVectorTemplate()))); + LongMaxVector toBitsVector() { + return (LongMaxVector) super.toBitsVectorTemplate(); } @Override @ForceInline - public VectorShuffle cast(VectorSpecies s) { - AbstractSpecies species = (AbstractSpecies) s; - if (length() != species.laneCount()) - throw new IllegalArgumentException("VectorShuffle length and species length differ"); - int[] shuffleArray = toArray(); - return s.shuffleFromArray(shuffleArray, 0).check(s); + LongVector toBitsVector0() { + return LongMaxVector.VSPECIES.dummyVector().vectorFactory(indices()); } + @Override @ForceInline + public int laneSource(int i) { + return (int)toBitsVector().lane(i); + } + @Override - public LongMaxShuffle rearrange(VectorShuffle shuffle) { - LongMaxShuffle s = (LongMaxShuffle) shuffle; - byte[] reorder1 = reorder(); - byte[] reorder2 = s.reorder(); - byte[] r = new byte[reorder1.length]; - for (int i = 0; i < reorder1.length; i++) { - int ssi = reorder2[i]; - r[i] = reorder1[ssi]; // throws on exceptional index + @ForceInline + public void intoArray(int[] a, int offset) { + VectorSpecies species = VectorSpecies.of( + int.class, + VectorShape.forBitSize(length() * Integer.SIZE)); + Vector v = toBitsVector(); + v.convertShape(VectorOperators.L2I, species, 0) + .reinterpretAsInts() + .intoArray(a, offset); + } + + private static long[] prepare(int[] indices, int offset) { + long[] a = new long[VLENGTH]; + for (int i = 0; i < VLENGTH; i++) { + int si = indices[offset + i]; + si = partiallyWrapIndex(si, VLENGTH); + a[i] = (long)si; + } + return a; + } + + private static long[] prepare(IntUnaryOperator f) { + long[] a = new long[VLENGTH]; + for (int i = 0; i < VLENGTH; i++) { + int si = f.applyAsInt(i); + si = partiallyWrapIndex(si, VLENGTH); + a[i] = (long)si; + } + return a; + } + + private static boolean indicesInRange(long[] indices) { + int length = indices.length; + for (long si : indices) { + if (si >= (long)length || si < (long)(-length)) { + boolean assertsEnabled = false; + assert(assertsEnabled = true); + if (assertsEnabled) { + String msg = ("index "+si+"out of range ["+length+"] in "+ + java.util.Arrays.toString(indices)); + throw new AssertionError(msg); + } + return false; + } } - return new LongMaxShuffle(r); + return true; } } diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/LongVector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/LongVector.java index 446d214a21bff..bbaa8fdb79923 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/LongVector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/LongVector.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, 2022, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2017, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -987,7 +987,7 @@ private static VectorBroadcastIntOp> broadcastIntOp // and broadcast, but it would be more surprising not to continue // the obvious pattern started by unary and binary. - /** + /** * {@inheritDoc} * @see #lanewise(VectorOperators.Ternary,long,long,VectorMask) * @see #lanewise(VectorOperators.Ternary,Vector,long,VectorMask) @@ -2331,8 +2331,8 @@ LongVector rearrangeTemplate(Class shuffletype, } @ForceInline - private final - VectorShuffle toShuffle0(LongSpecies dsp) { + final + VectorShuffle toShuffle0(AbstractSpecies dsp) { long[] a = toArray(); int[] sa = new int[a.length]; for (int i = 0; i < a.length; i++) { @@ -2341,18 +2341,6 @@ VectorShuffle toShuffle0(LongSpecies dsp) { return VectorShuffle.fromArray(dsp, sa, 0); } - /*package-private*/ - @ForceInline - final - VectorShuffle toShuffleTemplate(Class shuffleType) { - LongSpecies vsp = vspecies(); - return VectorSupport.convert(VectorSupport.VECTOR_OP_CAST, - getClass(), long.class, length(), - shuffleType, byte.class, length(), - this, vsp, - LongVector::toShuffle0); - } - /** * {@inheritDoc} * @since 19 @@ -3740,9 +3728,10 @@ static final class LongSpecies extends AbstractSpecies { private LongSpecies(VectorShape shape, Class vectorType, Class> maskType, + Class> shuffleType, Function vectorFactory) { super(shape, LaneType.of(long.class), - vectorType, maskType, + vectorType, maskType, shuffleType, vectorFactory); assert(this.elementSize() == Long.SIZE); } @@ -4009,6 +3998,7 @@ static LongSpecies species(VectorShape s) { = new LongSpecies(VectorShape.S_64_BIT, Long64Vector.class, Long64Vector.Long64Mask.class, + Long64Vector.Long64Shuffle.class, Long64Vector::new); /** Species representing {@link LongVector}s of {@link VectorShape#S_128_BIT VectorShape.S_128_BIT}. */ @@ -4016,6 +4006,7 @@ static LongSpecies species(VectorShape s) { = new LongSpecies(VectorShape.S_128_BIT, Long128Vector.class, Long128Vector.Long128Mask.class, + Long128Vector.Long128Shuffle.class, Long128Vector::new); /** Species representing {@link LongVector}s of {@link VectorShape#S_256_BIT VectorShape.S_256_BIT}. */ @@ -4023,6 +4014,7 @@ static LongSpecies species(VectorShape s) { = new LongSpecies(VectorShape.S_256_BIT, Long256Vector.class, Long256Vector.Long256Mask.class, + Long256Vector.Long256Shuffle.class, Long256Vector::new); /** Species representing {@link LongVector}s of {@link VectorShape#S_512_BIT VectorShape.S_512_BIT}. */ @@ -4030,6 +4022,7 @@ static LongSpecies species(VectorShape s) { = new LongSpecies(VectorShape.S_512_BIT, Long512Vector.class, Long512Vector.Long512Mask.class, + Long512Vector.Long512Shuffle.class, Long512Vector::new); /** Species representing {@link LongVector}s of {@link VectorShape#S_Max_BIT VectorShape.S_Max_BIT}. */ @@ -4037,6 +4030,7 @@ static LongSpecies species(VectorShape s) { = new LongSpecies(VectorShape.S_Max_BIT, LongMaxVector.class, LongMaxVector.LongMaxMask.class, + LongMaxVector.LongMaxShuffle.class, LongMaxVector::new); /** diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short128Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short128Vector.java index 1f587ca54a5f9..8796205b84d90 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short128Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short128Vector.java @@ -141,24 +141,9 @@ Short128Mask maskFromArray(boolean[] bits) { @ForceInline Short128Shuffle iotaShuffle() { return Short128Shuffle.IOTA; } - @ForceInline - Short128Shuffle iotaShuffle(int start, int step, boolean wrap) { - if (wrap) { - return (Short128Shuffle)VectorSupport.shuffleIota(ETYPE, Short128Shuffle.class, VSPECIES, VLENGTH, start, step, 1, - (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i*lstep + lstart, l)))); - } else { - return (Short128Shuffle)VectorSupport.shuffleIota(ETYPE, Short128Shuffle.class, VSPECIES, VLENGTH, start, step, 0, - (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (i*lstep + lstart))); - } - } - @Override @ForceInline - Short128Shuffle shuffleFromBytes(byte[] reorder) { return new Short128Shuffle(reorder); } - - @Override - @ForceInline - Short128Shuffle shuffleFromArray(int[] indexes, int i) { return new Short128Shuffle(indexes, i); } + Short128Shuffle shuffleFromArray(int[] indices, int i) { return new Short128Shuffle(indices, i); } @Override @ForceInline @@ -357,9 +342,11 @@ public final long reduceLanesToLong(VectorOperators.Associative op, return (long) super.reduceLanesTemplate(op, Short128Mask.class, (Short128Mask) m); // specialized } + @Override @ForceInline - public VectorShuffle toShuffle() { - return super.toShuffleTemplate(Short128Shuffle.class); // specialize + public final + VectorShuffle toShuffle(AbstractSpecies dsp) { + return super.toShuffleTemplate(dsp); } // Specialized unary testing @@ -796,23 +783,26 @@ static final class Short128Shuffle extends AbstractShuffle { static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM static final Class ETYPE = short.class; // used by the JVM - Short128Shuffle(byte[] reorder) { - super(VLENGTH, reorder); + Short128Shuffle(short[] indices) { + super(indices); + assert(VLENGTH == indices.length); + assert(indicesInRange(indices)); } - public Short128Shuffle(int[] reorder) { - super(VLENGTH, reorder); + Short128Shuffle(int[] indices, int i) { + this(prepare(indices, i)); } - public Short128Shuffle(int[] reorder, int i) { - super(VLENGTH, reorder, i); + Short128Shuffle(IntUnaryOperator fn) { + this(prepare(fn)); } - public Short128Shuffle(IntUnaryOperator fn) { - super(VLENGTH, fn); + short[] indices() { + return (short[])getPayload(); } @Override + @ForceInline public ShortSpecies vspecies() { return VSPECIES; } @@ -820,40 +810,77 @@ public ShortSpecies vspecies() { static { // There must be enough bits in the shuffle lanes to encode // VLENGTH valid indexes and VLENGTH exceptional ones. - assert(VLENGTH < Byte.MAX_VALUE); - assert(Byte.MIN_VALUE <= -VLENGTH); + assert(VLENGTH < Short.MAX_VALUE); + assert(Short.MIN_VALUE <= -VLENGTH); } static final Short128Shuffle IOTA = new Short128Shuffle(IDENTITY); @Override @ForceInline - public Short128Vector toVector() { - return VectorSupport.shuffleToVector(VCLASS, ETYPE, Short128Shuffle.class, this, VLENGTH, - (s) -> ((Short128Vector)(((AbstractShuffle)(s)).toVectorTemplate()))); + Short128Vector toBitsVector() { + return (Short128Vector) super.toBitsVectorTemplate(); } @Override @ForceInline - public VectorShuffle cast(VectorSpecies s) { - AbstractSpecies species = (AbstractSpecies) s; - if (length() != species.laneCount()) - throw new IllegalArgumentException("VectorShuffle length and species length differ"); - int[] shuffleArray = toArray(); - return s.shuffleFromArray(shuffleArray, 0).check(s); + ShortVector toBitsVector0() { + return Short128Vector.VSPECIES.dummyVector().vectorFactory(indices()); } + @Override @ForceInline + public int laneSource(int i) { + return (int)toBitsVector().lane(i); + } + @Override - public Short128Shuffle rearrange(VectorShuffle shuffle) { - Short128Shuffle s = (Short128Shuffle) shuffle; - byte[] reorder1 = reorder(); - byte[] reorder2 = s.reorder(); - byte[] r = new byte[reorder1.length]; - for (int i = 0; i < reorder1.length; i++) { - int ssi = reorder2[i]; - r[i] = reorder1[ssi]; // throws on exceptional index + @ForceInline + public void intoArray(int[] a, int offset) { + VectorSpecies species = IntVector.SPECIES_128; + Vector v = toBitsVector(); + v.convertShape(VectorOperators.S2I, species, 0) + .reinterpretAsInts() + .intoArray(a, offset); + v.convertShape(VectorOperators.S2I, species, 1) + .reinterpretAsInts() + .intoArray(a, offset + species.length()); + } + + private static short[] prepare(int[] indices, int offset) { + short[] a = new short[VLENGTH]; + for (int i = 0; i < VLENGTH; i++) { + int si = indices[offset + i]; + si = partiallyWrapIndex(si, VLENGTH); + a[i] = (short)si; + } + return a; + } + + private static short[] prepare(IntUnaryOperator f) { + short[] a = new short[VLENGTH]; + for (int i = 0; i < VLENGTH; i++) { + int si = f.applyAsInt(i); + si = partiallyWrapIndex(si, VLENGTH); + a[i] = (short)si; + } + return a; + } + + private static boolean indicesInRange(short[] indices) { + int length = indices.length; + for (short si : indices) { + if (si >= (short)length || si < (short)(-length)) { + boolean assertsEnabled = false; + assert(assertsEnabled = true); + if (assertsEnabled) { + String msg = ("index "+si+"out of range ["+length+"] in "+ + java.util.Arrays.toString(indices)); + throw new AssertionError(msg); + } + return false; + } } - return new Short128Shuffle(r); + return true; } } diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short256Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short256Vector.java index 5e259b70b591d..a9d5f3c47bdfa 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short256Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short256Vector.java @@ -141,24 +141,9 @@ Short256Mask maskFromArray(boolean[] bits) { @ForceInline Short256Shuffle iotaShuffle() { return Short256Shuffle.IOTA; } - @ForceInline - Short256Shuffle iotaShuffle(int start, int step, boolean wrap) { - if (wrap) { - return (Short256Shuffle)VectorSupport.shuffleIota(ETYPE, Short256Shuffle.class, VSPECIES, VLENGTH, start, step, 1, - (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i*lstep + lstart, l)))); - } else { - return (Short256Shuffle)VectorSupport.shuffleIota(ETYPE, Short256Shuffle.class, VSPECIES, VLENGTH, start, step, 0, - (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (i*lstep + lstart))); - } - } - @Override @ForceInline - Short256Shuffle shuffleFromBytes(byte[] reorder) { return new Short256Shuffle(reorder); } - - @Override - @ForceInline - Short256Shuffle shuffleFromArray(int[] indexes, int i) { return new Short256Shuffle(indexes, i); } + Short256Shuffle shuffleFromArray(int[] indices, int i) { return new Short256Shuffle(indices, i); } @Override @ForceInline @@ -357,9 +342,11 @@ public final long reduceLanesToLong(VectorOperators.Associative op, return (long) super.reduceLanesTemplate(op, Short256Mask.class, (Short256Mask) m); // specialized } + @Override @ForceInline - public VectorShuffle toShuffle() { - return super.toShuffleTemplate(Short256Shuffle.class); // specialize + public final + VectorShuffle toShuffle(AbstractSpecies dsp) { + return super.toShuffleTemplate(dsp); } // Specialized unary testing @@ -812,23 +799,26 @@ static final class Short256Shuffle extends AbstractShuffle { static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM static final Class ETYPE = short.class; // used by the JVM - Short256Shuffle(byte[] reorder) { - super(VLENGTH, reorder); + Short256Shuffle(short[] indices) { + super(indices); + assert(VLENGTH == indices.length); + assert(indicesInRange(indices)); } - public Short256Shuffle(int[] reorder) { - super(VLENGTH, reorder); + Short256Shuffle(int[] indices, int i) { + this(prepare(indices, i)); } - public Short256Shuffle(int[] reorder, int i) { - super(VLENGTH, reorder, i); + Short256Shuffle(IntUnaryOperator fn) { + this(prepare(fn)); } - public Short256Shuffle(IntUnaryOperator fn) { - super(VLENGTH, fn); + short[] indices() { + return (short[])getPayload(); } @Override + @ForceInline public ShortSpecies vspecies() { return VSPECIES; } @@ -836,40 +826,77 @@ public ShortSpecies vspecies() { static { // There must be enough bits in the shuffle lanes to encode // VLENGTH valid indexes and VLENGTH exceptional ones. - assert(VLENGTH < Byte.MAX_VALUE); - assert(Byte.MIN_VALUE <= -VLENGTH); + assert(VLENGTH < Short.MAX_VALUE); + assert(Short.MIN_VALUE <= -VLENGTH); } static final Short256Shuffle IOTA = new Short256Shuffle(IDENTITY); @Override @ForceInline - public Short256Vector toVector() { - return VectorSupport.shuffleToVector(VCLASS, ETYPE, Short256Shuffle.class, this, VLENGTH, - (s) -> ((Short256Vector)(((AbstractShuffle)(s)).toVectorTemplate()))); + Short256Vector toBitsVector() { + return (Short256Vector) super.toBitsVectorTemplate(); } @Override @ForceInline - public VectorShuffle cast(VectorSpecies s) { - AbstractSpecies species = (AbstractSpecies) s; - if (length() != species.laneCount()) - throw new IllegalArgumentException("VectorShuffle length and species length differ"); - int[] shuffleArray = toArray(); - return s.shuffleFromArray(shuffleArray, 0).check(s); + ShortVector toBitsVector0() { + return Short256Vector.VSPECIES.dummyVector().vectorFactory(indices()); } + @Override @ForceInline + public int laneSource(int i) { + return (int)toBitsVector().lane(i); + } + @Override - public Short256Shuffle rearrange(VectorShuffle shuffle) { - Short256Shuffle s = (Short256Shuffle) shuffle; - byte[] reorder1 = reorder(); - byte[] reorder2 = s.reorder(); - byte[] r = new byte[reorder1.length]; - for (int i = 0; i < reorder1.length; i++) { - int ssi = reorder2[i]; - r[i] = reorder1[ssi]; // throws on exceptional index + @ForceInline + public void intoArray(int[] a, int offset) { + VectorSpecies species = IntVector.SPECIES_256; + Vector v = toBitsVector(); + v.convertShape(VectorOperators.S2I, species, 0) + .reinterpretAsInts() + .intoArray(a, offset); + v.convertShape(VectorOperators.S2I, species, 1) + .reinterpretAsInts() + .intoArray(a, offset + species.length()); + } + + private static short[] prepare(int[] indices, int offset) { + short[] a = new short[VLENGTH]; + for (int i = 0; i < VLENGTH; i++) { + int si = indices[offset + i]; + si = partiallyWrapIndex(si, VLENGTH); + a[i] = (short)si; + } + return a; + } + + private static short[] prepare(IntUnaryOperator f) { + short[] a = new short[VLENGTH]; + for (int i = 0; i < VLENGTH; i++) { + int si = f.applyAsInt(i); + si = partiallyWrapIndex(si, VLENGTH); + a[i] = (short)si; + } + return a; + } + + private static boolean indicesInRange(short[] indices) { + int length = indices.length; + for (short si : indices) { + if (si >= (short)length || si < (short)(-length)) { + boolean assertsEnabled = false; + assert(assertsEnabled = true); + if (assertsEnabled) { + String msg = ("index "+si+"out of range ["+length+"] in "+ + java.util.Arrays.toString(indices)); + throw new AssertionError(msg); + } + return false; + } } - return new Short256Shuffle(r); + return true; } } diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short512Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short512Vector.java index a6cbfb4f8e489..e0dbb928316bf 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short512Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short512Vector.java @@ -141,24 +141,9 @@ Short512Mask maskFromArray(boolean[] bits) { @ForceInline Short512Shuffle iotaShuffle() { return Short512Shuffle.IOTA; } - @ForceInline - Short512Shuffle iotaShuffle(int start, int step, boolean wrap) { - if (wrap) { - return (Short512Shuffle)VectorSupport.shuffleIota(ETYPE, Short512Shuffle.class, VSPECIES, VLENGTH, start, step, 1, - (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i*lstep + lstart, l)))); - } else { - return (Short512Shuffle)VectorSupport.shuffleIota(ETYPE, Short512Shuffle.class, VSPECIES, VLENGTH, start, step, 0, - (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (i*lstep + lstart))); - } - } - @Override @ForceInline - Short512Shuffle shuffleFromBytes(byte[] reorder) { return new Short512Shuffle(reorder); } - - @Override - @ForceInline - Short512Shuffle shuffleFromArray(int[] indexes, int i) { return new Short512Shuffle(indexes, i); } + Short512Shuffle shuffleFromArray(int[] indices, int i) { return new Short512Shuffle(indices, i); } @Override @ForceInline @@ -357,9 +342,11 @@ public final long reduceLanesToLong(VectorOperators.Associative op, return (long) super.reduceLanesTemplate(op, Short512Mask.class, (Short512Mask) m); // specialized } + @Override @ForceInline - public VectorShuffle toShuffle() { - return super.toShuffleTemplate(Short512Shuffle.class); // specialize + public final + VectorShuffle toShuffle(AbstractSpecies dsp) { + return super.toShuffleTemplate(dsp); } // Specialized unary testing @@ -844,23 +831,26 @@ static final class Short512Shuffle extends AbstractShuffle { static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM static final Class ETYPE = short.class; // used by the JVM - Short512Shuffle(byte[] reorder) { - super(VLENGTH, reorder); + Short512Shuffle(short[] indices) { + super(indices); + assert(VLENGTH == indices.length); + assert(indicesInRange(indices)); } - public Short512Shuffle(int[] reorder) { - super(VLENGTH, reorder); + Short512Shuffle(int[] indices, int i) { + this(prepare(indices, i)); } - public Short512Shuffle(int[] reorder, int i) { - super(VLENGTH, reorder, i); + Short512Shuffle(IntUnaryOperator fn) { + this(prepare(fn)); } - public Short512Shuffle(IntUnaryOperator fn) { - super(VLENGTH, fn); + short[] indices() { + return (short[])getPayload(); } @Override + @ForceInline public ShortSpecies vspecies() { return VSPECIES; } @@ -868,40 +858,77 @@ public ShortSpecies vspecies() { static { // There must be enough bits in the shuffle lanes to encode // VLENGTH valid indexes and VLENGTH exceptional ones. - assert(VLENGTH < Byte.MAX_VALUE); - assert(Byte.MIN_VALUE <= -VLENGTH); + assert(VLENGTH < Short.MAX_VALUE); + assert(Short.MIN_VALUE <= -VLENGTH); } static final Short512Shuffle IOTA = new Short512Shuffle(IDENTITY); @Override @ForceInline - public Short512Vector toVector() { - return VectorSupport.shuffleToVector(VCLASS, ETYPE, Short512Shuffle.class, this, VLENGTH, - (s) -> ((Short512Vector)(((AbstractShuffle)(s)).toVectorTemplate()))); + Short512Vector toBitsVector() { + return (Short512Vector) super.toBitsVectorTemplate(); } @Override @ForceInline - public VectorShuffle cast(VectorSpecies s) { - AbstractSpecies species = (AbstractSpecies) s; - if (length() != species.laneCount()) - throw new IllegalArgumentException("VectorShuffle length and species length differ"); - int[] shuffleArray = toArray(); - return s.shuffleFromArray(shuffleArray, 0).check(s); + ShortVector toBitsVector0() { + return Short512Vector.VSPECIES.dummyVector().vectorFactory(indices()); } + @Override @ForceInline + public int laneSource(int i) { + return (int)toBitsVector().lane(i); + } + @Override - public Short512Shuffle rearrange(VectorShuffle shuffle) { - Short512Shuffle s = (Short512Shuffle) shuffle; - byte[] reorder1 = reorder(); - byte[] reorder2 = s.reorder(); - byte[] r = new byte[reorder1.length]; - for (int i = 0; i < reorder1.length; i++) { - int ssi = reorder2[i]; - r[i] = reorder1[ssi]; // throws on exceptional index + @ForceInline + public void intoArray(int[] a, int offset) { + VectorSpecies species = IntVector.SPECIES_512; + Vector v = toBitsVector(); + v.convertShape(VectorOperators.S2I, species, 0) + .reinterpretAsInts() + .intoArray(a, offset); + v.convertShape(VectorOperators.S2I, species, 1) + .reinterpretAsInts() + .intoArray(a, offset + species.length()); + } + + private static short[] prepare(int[] indices, int offset) { + short[] a = new short[VLENGTH]; + for (int i = 0; i < VLENGTH; i++) { + int si = indices[offset + i]; + si = partiallyWrapIndex(si, VLENGTH); + a[i] = (short)si; + } + return a; + } + + private static short[] prepare(IntUnaryOperator f) { + short[] a = new short[VLENGTH]; + for (int i = 0; i < VLENGTH; i++) { + int si = f.applyAsInt(i); + si = partiallyWrapIndex(si, VLENGTH); + a[i] = (short)si; + } + return a; + } + + private static boolean indicesInRange(short[] indices) { + int length = indices.length; + for (short si : indices) { + if (si >= (short)length || si < (short)(-length)) { + boolean assertsEnabled = false; + assert(assertsEnabled = true); + if (assertsEnabled) { + String msg = ("index "+si+"out of range ["+length+"] in "+ + java.util.Arrays.toString(indices)); + throw new AssertionError(msg); + } + return false; + } } - return new Short512Shuffle(r); + return true; } } diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short64Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short64Vector.java index b2a2ec119524d..8a0a1f616d68c 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short64Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short64Vector.java @@ -141,24 +141,9 @@ Short64Mask maskFromArray(boolean[] bits) { @ForceInline Short64Shuffle iotaShuffle() { return Short64Shuffle.IOTA; } - @ForceInline - Short64Shuffle iotaShuffle(int start, int step, boolean wrap) { - if (wrap) { - return (Short64Shuffle)VectorSupport.shuffleIota(ETYPE, Short64Shuffle.class, VSPECIES, VLENGTH, start, step, 1, - (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i*lstep + lstart, l)))); - } else { - return (Short64Shuffle)VectorSupport.shuffleIota(ETYPE, Short64Shuffle.class, VSPECIES, VLENGTH, start, step, 0, - (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (i*lstep + lstart))); - } - } - @Override @ForceInline - Short64Shuffle shuffleFromBytes(byte[] reorder) { return new Short64Shuffle(reorder); } - - @Override - @ForceInline - Short64Shuffle shuffleFromArray(int[] indexes, int i) { return new Short64Shuffle(indexes, i); } + Short64Shuffle shuffleFromArray(int[] indices, int i) { return new Short64Shuffle(indices, i); } @Override @ForceInline @@ -357,9 +342,11 @@ public final long reduceLanesToLong(VectorOperators.Associative op, return (long) super.reduceLanesTemplate(op, Short64Mask.class, (Short64Mask) m); // specialized } + @Override @ForceInline - public VectorShuffle toShuffle() { - return super.toShuffleTemplate(Short64Shuffle.class); // specialize + public final + VectorShuffle toShuffle(AbstractSpecies dsp) { + return super.toShuffleTemplate(dsp); } // Specialized unary testing @@ -788,23 +775,26 @@ static final class Short64Shuffle extends AbstractShuffle { static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM static final Class ETYPE = short.class; // used by the JVM - Short64Shuffle(byte[] reorder) { - super(VLENGTH, reorder); + Short64Shuffle(short[] indices) { + super(indices); + assert(VLENGTH == indices.length); + assert(indicesInRange(indices)); } - public Short64Shuffle(int[] reorder) { - super(VLENGTH, reorder); + Short64Shuffle(int[] indices, int i) { + this(prepare(indices, i)); } - public Short64Shuffle(int[] reorder, int i) { - super(VLENGTH, reorder, i); + Short64Shuffle(IntUnaryOperator fn) { + this(prepare(fn)); } - public Short64Shuffle(IntUnaryOperator fn) { - super(VLENGTH, fn); + short[] indices() { + return (short[])getPayload(); } @Override + @ForceInline public ShortSpecies vspecies() { return VSPECIES; } @@ -812,40 +802,77 @@ public ShortSpecies vspecies() { static { // There must be enough bits in the shuffle lanes to encode // VLENGTH valid indexes and VLENGTH exceptional ones. - assert(VLENGTH < Byte.MAX_VALUE); - assert(Byte.MIN_VALUE <= -VLENGTH); + assert(VLENGTH < Short.MAX_VALUE); + assert(Short.MIN_VALUE <= -VLENGTH); } static final Short64Shuffle IOTA = new Short64Shuffle(IDENTITY); @Override @ForceInline - public Short64Vector toVector() { - return VectorSupport.shuffleToVector(VCLASS, ETYPE, Short64Shuffle.class, this, VLENGTH, - (s) -> ((Short64Vector)(((AbstractShuffle)(s)).toVectorTemplate()))); + Short64Vector toBitsVector() { + return (Short64Vector) super.toBitsVectorTemplate(); } @Override @ForceInline - public VectorShuffle cast(VectorSpecies s) { - AbstractSpecies species = (AbstractSpecies) s; - if (length() != species.laneCount()) - throw new IllegalArgumentException("VectorShuffle length and species length differ"); - int[] shuffleArray = toArray(); - return s.shuffleFromArray(shuffleArray, 0).check(s); + ShortVector toBitsVector0() { + return Short64Vector.VSPECIES.dummyVector().vectorFactory(indices()); } + @Override @ForceInline + public int laneSource(int i) { + return (int)toBitsVector().lane(i); + } + @Override - public Short64Shuffle rearrange(VectorShuffle shuffle) { - Short64Shuffle s = (Short64Shuffle) shuffle; - byte[] reorder1 = reorder(); - byte[] reorder2 = s.reorder(); - byte[] r = new byte[reorder1.length]; - for (int i = 0; i < reorder1.length; i++) { - int ssi = reorder2[i]; - r[i] = reorder1[ssi]; // throws on exceptional index + @ForceInline + public void intoArray(int[] a, int offset) { + VectorSpecies species = IntVector.SPECIES_64; + Vector v = toBitsVector(); + v.convertShape(VectorOperators.S2I, species, 0) + .reinterpretAsInts() + .intoArray(a, offset); + v.convertShape(VectorOperators.S2I, species, 1) + .reinterpretAsInts() + .intoArray(a, offset + species.length()); + } + + private static short[] prepare(int[] indices, int offset) { + short[] a = new short[VLENGTH]; + for (int i = 0; i < VLENGTH; i++) { + int si = indices[offset + i]; + si = partiallyWrapIndex(si, VLENGTH); + a[i] = (short)si; + } + return a; + } + + private static short[] prepare(IntUnaryOperator f) { + short[] a = new short[VLENGTH]; + for (int i = 0; i < VLENGTH; i++) { + int si = f.applyAsInt(i); + si = partiallyWrapIndex(si, VLENGTH); + a[i] = (short)si; + } + return a; + } + + private static boolean indicesInRange(short[] indices) { + int length = indices.length; + for (short si : indices) { + if (si >= (short)length || si < (short)(-length)) { + boolean assertsEnabled = false; + assert(assertsEnabled = true); + if (assertsEnabled) { + String msg = ("index "+si+"out of range ["+length+"] in "+ + java.util.Arrays.toString(indices)); + throw new AssertionError(msg); + } + return false; + } } - return new Short64Shuffle(r); + return true; } } diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ShortMaxVector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ShortMaxVector.java index eaf99f61896b5..110c792b85d2a 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ShortMaxVector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ShortMaxVector.java @@ -141,24 +141,9 @@ ShortMaxMask maskFromArray(boolean[] bits) { @ForceInline ShortMaxShuffle iotaShuffle() { return ShortMaxShuffle.IOTA; } - @ForceInline - ShortMaxShuffle iotaShuffle(int start, int step, boolean wrap) { - if (wrap) { - return (ShortMaxShuffle)VectorSupport.shuffleIota(ETYPE, ShortMaxShuffle.class, VSPECIES, VLENGTH, start, step, 1, - (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i*lstep + lstart, l)))); - } else { - return (ShortMaxShuffle)VectorSupport.shuffleIota(ETYPE, ShortMaxShuffle.class, VSPECIES, VLENGTH, start, step, 0, - (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (i*lstep + lstart))); - } - } - @Override @ForceInline - ShortMaxShuffle shuffleFromBytes(byte[] reorder) { return new ShortMaxShuffle(reorder); } - - @Override - @ForceInline - ShortMaxShuffle shuffleFromArray(int[] indexes, int i) { return new ShortMaxShuffle(indexes, i); } + ShortMaxShuffle shuffleFromArray(int[] indices, int i) { return new ShortMaxShuffle(indices, i); } @Override @ForceInline @@ -357,9 +342,11 @@ public final long reduceLanesToLong(VectorOperators.Associative op, return (long) super.reduceLanesTemplate(op, ShortMaxMask.class, (ShortMaxMask) m); // specialized } + @Override @ForceInline - public VectorShuffle toShuffle() { - return super.toShuffleTemplate(ShortMaxShuffle.class); // specialize + public final + VectorShuffle toShuffle(AbstractSpecies dsp) { + return super.toShuffleTemplate(dsp); } // Specialized unary testing @@ -782,23 +769,26 @@ static final class ShortMaxShuffle extends AbstractShuffle { static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM static final Class ETYPE = short.class; // used by the JVM - ShortMaxShuffle(byte[] reorder) { - super(VLENGTH, reorder); + ShortMaxShuffle(short[] indices) { + super(indices); + assert(VLENGTH == indices.length); + assert(indicesInRange(indices)); } - public ShortMaxShuffle(int[] reorder) { - super(VLENGTH, reorder); + ShortMaxShuffle(int[] indices, int i) { + this(prepare(indices, i)); } - public ShortMaxShuffle(int[] reorder, int i) { - super(VLENGTH, reorder, i); + ShortMaxShuffle(IntUnaryOperator fn) { + this(prepare(fn)); } - public ShortMaxShuffle(IntUnaryOperator fn) { - super(VLENGTH, fn); + short[] indices() { + return (short[])getPayload(); } @Override + @ForceInline public ShortSpecies vspecies() { return VSPECIES; } @@ -806,40 +796,77 @@ public ShortSpecies vspecies() { static { // There must be enough bits in the shuffle lanes to encode // VLENGTH valid indexes and VLENGTH exceptional ones. - assert(VLENGTH < Byte.MAX_VALUE); - assert(Byte.MIN_VALUE <= -VLENGTH); + assert(VLENGTH < Short.MAX_VALUE); + assert(Short.MIN_VALUE <= -VLENGTH); } static final ShortMaxShuffle IOTA = new ShortMaxShuffle(IDENTITY); @Override @ForceInline - public ShortMaxVector toVector() { - return VectorSupport.shuffleToVector(VCLASS, ETYPE, ShortMaxShuffle.class, this, VLENGTH, - (s) -> ((ShortMaxVector)(((AbstractShuffle)(s)).toVectorTemplate()))); + ShortMaxVector toBitsVector() { + return (ShortMaxVector) super.toBitsVectorTemplate(); } @Override @ForceInline - public VectorShuffle cast(VectorSpecies s) { - AbstractSpecies species = (AbstractSpecies) s; - if (length() != species.laneCount()) - throw new IllegalArgumentException("VectorShuffle length and species length differ"); - int[] shuffleArray = toArray(); - return s.shuffleFromArray(shuffleArray, 0).check(s); + ShortVector toBitsVector0() { + return ShortMaxVector.VSPECIES.dummyVector().vectorFactory(indices()); } + @Override @ForceInline + public int laneSource(int i) { + return (int)toBitsVector().lane(i); + } + @Override - public ShortMaxShuffle rearrange(VectorShuffle shuffle) { - ShortMaxShuffle s = (ShortMaxShuffle) shuffle; - byte[] reorder1 = reorder(); - byte[] reorder2 = s.reorder(); - byte[] r = new byte[reorder1.length]; - for (int i = 0; i < reorder1.length; i++) { - int ssi = reorder2[i]; - r[i] = reorder1[ssi]; // throws on exceptional index + @ForceInline + public void intoArray(int[] a, int offset) { + VectorSpecies species = IntVector.SPECIES_MAX; + Vector v = toBitsVector(); + v.convertShape(VectorOperators.S2I, species, 0) + .reinterpretAsInts() + .intoArray(a, offset); + v.convertShape(VectorOperators.S2I, species, 1) + .reinterpretAsInts() + .intoArray(a, offset + species.length()); + } + + private static short[] prepare(int[] indices, int offset) { + short[] a = new short[VLENGTH]; + for (int i = 0; i < VLENGTH; i++) { + int si = indices[offset + i]; + si = partiallyWrapIndex(si, VLENGTH); + a[i] = (short)si; + } + return a; + } + + private static short[] prepare(IntUnaryOperator f) { + short[] a = new short[VLENGTH]; + for (int i = 0; i < VLENGTH; i++) { + int si = f.applyAsInt(i); + si = partiallyWrapIndex(si, VLENGTH); + a[i] = (short)si; + } + return a; + } + + private static boolean indicesInRange(short[] indices) { + int length = indices.length; + for (short si : indices) { + if (si >= (short)length || si < (short)(-length)) { + boolean assertsEnabled = false; + assert(assertsEnabled = true); + if (assertsEnabled) { + String msg = ("index "+si+"out of range ["+length+"] in "+ + java.util.Arrays.toString(indices)); + throw new AssertionError(msg); + } + return false; + } } - return new ShortMaxShuffle(r); + return true; } } diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ShortVector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ShortVector.java index af7ef119a92df..8b08f766588df 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ShortVector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ShortVector.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, 2022, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2017, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -1071,7 +1071,7 @@ private static VectorBroadcastIntOp> broadcastInt // and broadcast, but it would be more surprising not to continue // the obvious pattern started by unary and binary. - /** + /** * {@inheritDoc} * @see #lanewise(VectorOperators.Ternary,short,short,VectorMask) * @see #lanewise(VectorOperators.Ternary,Vector,short,VectorMask) @@ -2481,8 +2481,8 @@ ShortVector rearrangeTemplate(Class shuffletype, } @ForceInline - private final - VectorShuffle toShuffle0(ShortSpecies dsp) { + final + VectorShuffle toShuffle0(AbstractSpecies dsp) { short[] a = toArray(); int[] sa = new int[a.length]; for (int i = 0; i < a.length; i++) { @@ -2491,18 +2491,6 @@ VectorShuffle toShuffle0(ShortSpecies dsp) { return VectorShuffle.fromArray(dsp, sa, 0); } - /*package-private*/ - @ForceInline - final - VectorShuffle toShuffleTemplate(Class shuffleType) { - ShortSpecies vsp = vspecies(); - return VectorSupport.convert(VectorSupport.VECTOR_OP_CAST, - getClass(), short.class, length(), - shuffleType, byte.class, length(), - this, vsp, - ShortVector::toShuffle0); - } - /** * {@inheritDoc} * @since 19 @@ -4090,9 +4078,10 @@ static final class ShortSpecies extends AbstractSpecies { private ShortSpecies(VectorShape shape, Class vectorType, Class> maskType, + Class> shuffleType, Function vectorFactory) { super(shape, LaneType.of(short.class), - vectorType, maskType, + vectorType, maskType, shuffleType, vectorFactory); assert(this.elementSize() == Short.SIZE); } @@ -4368,6 +4357,7 @@ static ShortSpecies species(VectorShape s) { = new ShortSpecies(VectorShape.S_64_BIT, Short64Vector.class, Short64Vector.Short64Mask.class, + Short64Vector.Short64Shuffle.class, Short64Vector::new); /** Species representing {@link ShortVector}s of {@link VectorShape#S_128_BIT VectorShape.S_128_BIT}. */ @@ -4375,6 +4365,7 @@ static ShortSpecies species(VectorShape s) { = new ShortSpecies(VectorShape.S_128_BIT, Short128Vector.class, Short128Vector.Short128Mask.class, + Short128Vector.Short128Shuffle.class, Short128Vector::new); /** Species representing {@link ShortVector}s of {@link VectorShape#S_256_BIT VectorShape.S_256_BIT}. */ @@ -4382,6 +4373,7 @@ static ShortSpecies species(VectorShape s) { = new ShortSpecies(VectorShape.S_256_BIT, Short256Vector.class, Short256Vector.Short256Mask.class, + Short256Vector.Short256Shuffle.class, Short256Vector::new); /** Species representing {@link ShortVector}s of {@link VectorShape#S_512_BIT VectorShape.S_512_BIT}. */ @@ -4389,6 +4381,7 @@ static ShortSpecies species(VectorShape s) { = new ShortSpecies(VectorShape.S_512_BIT, Short512Vector.class, Short512Vector.Short512Mask.class, + Short512Vector.Short512Shuffle.class, Short512Vector::new); /** Species representing {@link ShortVector}s of {@link VectorShape#S_Max_BIT VectorShape.S_Max_BIT}. */ @@ -4396,6 +4389,7 @@ static ShortSpecies species(VectorShape s) { = new ShortSpecies(VectorShape.S_Max_BIT, ShortMaxVector.class, ShortMaxVector.ShortMaxMask.class, + ShortMaxVector.ShortMaxShuffle.class, ShortMaxVector::new); /** diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/VectorShape.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/VectorShape.java index 1ffbcef821afb..89e0d38bcb06c 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/VectorShape.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/VectorShape.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2017, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -124,6 +124,7 @@ VectorSpecies withLanes(Class elementType) { * @throws IllegalArgumentException if no such vector shape exists * @see #vectorBitSize() */ + @ForceInline public static VectorShape forBitSize(int bitSize) { switch (bitSize) { case 64: diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/VectorShuffle.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/VectorShuffle.java index b046874c39052..bc1780a81acc0 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/VectorShuffle.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/VectorShuffle.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2017, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -133,8 +133,8 @@ */ @SuppressWarnings("exports") public abstract class VectorShuffle extends jdk.internal.vm.vector.VectorSupport.VectorShuffle { - VectorShuffle(byte[] reorder) { - super(reorder); + VectorShuffle(Object indices) { + super(indices); } /** @@ -556,7 +556,7 @@ private static int unzipIndex(int i, int vlen, int part) { * @param i the lane index * @return the {@code int} lane element at lane index {@code i} */ - public int laneSource(int i) { return toArray()[i]; } + public abstract int laneSource(int i); /** * Rearranges the lane elements of this shuffle selecting lane indexes diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/VectorSpecies.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/VectorSpecies.java index 6cfe5f6d9d93e..03cb0ae559e70 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/VectorSpecies.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/VectorSpecies.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, 2022, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2017, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -24,6 +24,8 @@ */ package jdk.incubator.vector; +import jdk.internal.vm.annotation.ForceInline; + import java.lang.foreign.MemorySegment; import java.nio.ByteOrder; @@ -342,6 +344,7 @@ public interface VectorSpecies { * @see #withLanes(Class) * @see #withShape(VectorShape) */ + @ForceInline static VectorSpecies of(Class elementType, VectorShape shape) { LaneType laneType = LaneType.of(elementType); return AbstractSpecies.findSpecies(elementType, laneType, shape); @@ -367,6 +370,7 @@ static VectorSpecies of(Class elementType, VectorShape shape) { * or if the given type is not a valid {@code ETYPE} * @see VectorSpecies#ofPreferred(Class) */ + @ForceInline static VectorSpecies ofLargestShape(Class etype) { return VectorSpecies.of(etype, VectorShape.largestShapeFor(etype)); } @@ -410,6 +414,7 @@ static VectorSpecies ofLargestShape(Class etype) { * @see VectorShape#preferredShape() * @see VectorSpecies#ofLargestShape(Class) */ + @ForceInline public static VectorSpecies ofPreferred(Class etype) { return of(etype, VectorShape.preferredShape()); } @@ -432,6 +437,7 @@ public static VectorSpecies ofPreferred(Class etype) { * if the given {@code elementType} argument is not * a valid vector {@code ETYPE} */ + @ForceInline static int elementSize(Class elementType) { return LaneType.of(elementType).elementSize; } diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/X-Vector.java.template b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/X-Vector.java.template index a4b4750f2523e..af7ff88f48986 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/X-Vector.java.template +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/X-Vector.java.template @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, 2022, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2017, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -1204,7 +1204,7 @@ public abstract class $abstractvectortype$ extends AbstractVector<$Boxtype$> { // and broadcast, but it would be more surprising not to continue // the obvious pattern started by unary and binary. - /** + /** * {@inheritDoc} * @see #lanewise(VectorOperators.Ternary,$type$,$type$,VectorMask) * @see #lanewise(VectorOperators.Ternary,Vector,$type$,VectorMask) @@ -2857,8 +2857,8 @@ public abstract class $abstractvectortype$ extends AbstractVector<$Boxtype$> { } @ForceInline - private final - VectorShuffle<$Boxtype$> toShuffle0($Type$Species dsp) { + final + VectorShuffle toShuffle0(AbstractSpecies dsp) { $type$[] a = toArray(); int[] sa = new int[a.length]; for (int i = 0; i < a.length; i++) { @@ -2867,18 +2867,6 @@ public abstract class $abstractvectortype$ extends AbstractVector<$Boxtype$> { return VectorShuffle.fromArray(dsp, sa, 0); } - /*package-private*/ - @ForceInline - final - VectorShuffle<$Boxtype$> toShuffleTemplate(Class shuffleType) { - $Type$Species vsp = vspecies(); - return VectorSupport.convert(VectorSupport.VECTOR_OP_CAST, - getClass(), $type$.class, length(), - shuffleType, byte.class, length(), - this, vsp, - $Type$Vector::toShuffle0); - } - /** * {@inheritDoc} * @since 19 @@ -5348,9 +5336,10 @@ public abstract class $abstractvectortype$ extends AbstractVector<$Boxtype$> { private $Type$Species(VectorShape shape, Class vectorType, Class> maskType, + Class> shuffleType, Function vectorFactory) { super(shape, LaneType.of($type$.class), - vectorType, maskType, + vectorType, maskType, shuffleType, vectorFactory); assert(this.elementSize() == $Boxtype$.SIZE); } @@ -5633,6 +5622,7 @@ public abstract class $abstractvectortype$ extends AbstractVector<$Boxtype$> { = new $Type$Species(VectorShape.S_64_BIT, $Type$64Vector.class, $Type$64Vector.$Type$64Mask.class, + $Type$64Vector.$Type$64Shuffle.class, $Type$64Vector::new); /** Species representing {@link $Type$Vector}s of {@link VectorShape#S_128_BIT VectorShape.S_128_BIT}. */ @@ -5640,6 +5630,7 @@ public abstract class $abstractvectortype$ extends AbstractVector<$Boxtype$> { = new $Type$Species(VectorShape.S_128_BIT, $Type$128Vector.class, $Type$128Vector.$Type$128Mask.class, + $Type$128Vector.$Type$128Shuffle.class, $Type$128Vector::new); /** Species representing {@link $Type$Vector}s of {@link VectorShape#S_256_BIT VectorShape.S_256_BIT}. */ @@ -5647,6 +5638,7 @@ public abstract class $abstractvectortype$ extends AbstractVector<$Boxtype$> { = new $Type$Species(VectorShape.S_256_BIT, $Type$256Vector.class, $Type$256Vector.$Type$256Mask.class, + $Type$256Vector.$Type$256Shuffle.class, $Type$256Vector::new); /** Species representing {@link $Type$Vector}s of {@link VectorShape#S_512_BIT VectorShape.S_512_BIT}. */ @@ -5654,6 +5646,7 @@ public abstract class $abstractvectortype$ extends AbstractVector<$Boxtype$> { = new $Type$Species(VectorShape.S_512_BIT, $Type$512Vector.class, $Type$512Vector.$Type$512Mask.class, + $Type$512Vector.$Type$512Shuffle.class, $Type$512Vector::new); /** Species representing {@link $Type$Vector}s of {@link VectorShape#S_Max_BIT VectorShape.S_Max_BIT}. */ @@ -5661,6 +5654,7 @@ public abstract class $abstractvectortype$ extends AbstractVector<$Boxtype$> { = new $Type$Species(VectorShape.S_Max_BIT, $Type$MaxVector.class, $Type$MaxVector.$Type$MaxMask.class, + $Type$MaxVector.$Type$MaxShuffle.class, $Type$MaxVector::new); /** diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/X-VectorBits.java.template b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/X-VectorBits.java.template index 14ba8d34161f1..406f94f059467 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/X-VectorBits.java.template +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/X-VectorBits.java.template @@ -143,24 +143,9 @@ final class $vectortype$ extends $abstractvectortype$ { @ForceInline $shuffletype$ iotaShuffle() { return $shuffletype$.IOTA; } - @ForceInline - $shuffletype$ iotaShuffle(int start, int step, boolean wrap) { - if (wrap) { - return ($shuffletype$)VectorSupport.shuffleIota(ETYPE, $shuffletype$.class, VSPECIES, VLENGTH, start, step, 1, - (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i*lstep + lstart, l)))); - } else { - return ($shuffletype$)VectorSupport.shuffleIota(ETYPE, $shuffletype$.class, VSPECIES, VLENGTH, start, step, 0, - (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (i*lstep + lstart))); - } - } - - @Override - @ForceInline - $shuffletype$ shuffleFromBytes(byte[] reorder) { return new $shuffletype$(reorder); } - @Override @ForceInline - $shuffletype$ shuffleFromArray(int[] indexes, int i) { return new $shuffletype$(indexes, i); } + $shuffletype$ shuffleFromArray(int[] indices, int i) { return new $shuffletype$(indices, i); } @Override @ForceInline @@ -361,9 +346,11 @@ final class $vectortype$ extends $abstractvectortype$ { return (long) super.reduceLanesTemplate(op, $masktype$.class, ($masktype$) m); // specialized } + @Override @ForceInline - public VectorShuffle<$Boxtype$> toShuffle() { - return super.toShuffleTemplate($shuffletype$.class); // specialize + public final + VectorShuffle toShuffle(AbstractSpecies dsp) { + return super.toShuffleTemplate(dsp); } // Specialized unary testing @@ -1068,25 +1055,28 @@ final class $vectortype$ extends $abstractvectortype$ { static final class $shuffletype$ extends AbstractShuffle<$Boxtype$> { static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM - static final Class<$Boxtype$> ETYPE = $type$.class; // used by the JVM + static final Class<$Boxbitstype$> ETYPE = $bitstype$.class; // used by the JVM - $shuffletype$(byte[] reorder) { - super(VLENGTH, reorder); + $shuffletype$($bitstype$[] indices) { + super(indices); + assert(VLENGTH == indices.length); + assert(indicesInRange(indices)); } - public $shuffletype$(int[] reorder) { - super(VLENGTH, reorder); + $shuffletype$(int[] indices, int i) { + this(prepare(indices, i)); } - public $shuffletype$(int[] reorder, int i) { - super(VLENGTH, reorder, i); + $shuffletype$(IntUnaryOperator fn) { + this(prepare(fn)); } - public $shuffletype$(IntUnaryOperator fn) { - super(VLENGTH, fn); + $bitstype$[] indices() { + return ($bitstype$[])getPayload(); } @Override + @ForceInline public $Type$Species vspecies() { return VSPECIES; } @@ -1094,40 +1084,112 @@ final class $vectortype$ extends $abstractvectortype$ { static { // There must be enough bits in the shuffle lanes to encode // VLENGTH valid indexes and VLENGTH exceptional ones. - assert(VLENGTH < Byte.MAX_VALUE); - assert(Byte.MIN_VALUE <= -VLENGTH); + assert(VLENGTH < $Boxbitstype$.MAX_VALUE); + assert($Boxbitstype$.MIN_VALUE <= -VLENGTH); } static final $shuffletype$ IOTA = new $shuffletype$(IDENTITY); @Override @ForceInline - public $vectortype$ toVector() { - return VectorSupport.shuffleToVector(VCLASS, ETYPE, $shuffletype$.class, this, VLENGTH, - (s) -> (($vectortype$)(((AbstractShuffle<$Boxtype$>)(s)).toVectorTemplate()))); + $bitsvectortype$ toBitsVector() { + return ($bitsvectortype$) super.toBitsVectorTemplate(); } @Override @ForceInline - public VectorShuffle cast(VectorSpecies s) { - AbstractSpecies species = (AbstractSpecies) s; - if (length() != species.laneCount()) - throw new IllegalArgumentException("VectorShuffle length and species length differ"); - int[] shuffleArray = toArray(); - return s.shuffleFromArray(shuffleArray, 0).check(s); + $Bitstype$Vector toBitsVector0() { + return $bitsvectortype$.VSPECIES.dummyVector().vectorFactory(indices()); } + @Override @ForceInline + public int laneSource(int i) { + return (int)toBitsVector().lane(i); + } + @Override - public $shuffletype$ rearrange(VectorShuffle<$Boxtype$> shuffle) { - $shuffletype$ s = ($shuffletype$) shuffle; - byte[] reorder1 = reorder(); - byte[] reorder2 = s.reorder(); - byte[] r = new byte[reorder1.length]; - for (int i = 0; i < reorder1.length; i++) { - int ssi = reorder2[i]; - r[i] = reorder1[ssi]; // throws on exceptional index + @ForceInline + public void intoArray(int[] a, int offset) { +#if[byte] + VectorSpecies species = IntVector.SPECIES_$BITS$; + Vector v = toBitsVector(); + v.convertShape(VectorOperators.B2I, species, 0) + .reinterpretAsInts() + .intoArray(a, offset); + v.convertShape(VectorOperators.B2I, species, 1) + .reinterpretAsInts() + .intoArray(a, offset + species.length()); + v.convertShape(VectorOperators.B2I, species, 2) + .reinterpretAsInts() + .intoArray(a, offset + species.length() * 2); + v.convertShape(VectorOperators.B2I, species, 3) + .reinterpretAsInts() + .intoArray(a, offset + species.length() * 3); +#end[byte] +#if[short] + VectorSpecies species = IntVector.SPECIES_$BITS$; + Vector v = toBitsVector(); + v.convertShape(VectorOperators.S2I, species, 0) + .reinterpretAsInts() + .intoArray(a, offset); + v.convertShape(VectorOperators.S2I, species, 1) + .reinterpretAsInts() + .intoArray(a, offset + species.length()); +#end[short] +#if[intOrFloat] + toBitsVector().intoArray(a, offset); +#end[intOrFloat] +#if[longOrDouble] +#if[!1L] + VectorSpecies species = VectorSpecies.of( + int.class, + VectorShape.forBitSize(length() * Integer.SIZE)); + Vector v = toBitsVector(); + v.convertShape(VectorOperators.L2I, species, 0) + .reinterpretAsInts() + .intoArray(a, offset); +#end[!1L] +#if[1L] + a[offset] = laneSource(0); +#end[1L] +#end[longOrDouble] + } + + private static $bitstype$[] prepare(int[] indices, int offset) { + $bitstype$[] a = new $bitstype$[VLENGTH]; + for (int i = 0; i < VLENGTH; i++) { + int si = indices[offset + i]; + si = partiallyWrapIndex(si, VLENGTH); + a[i] = ($bitstype$)si; + } + return a; + } + + private static $bitstype$[] prepare(IntUnaryOperator f) { + $bitstype$[] a = new $bitstype$[VLENGTH]; + for (int i = 0; i < VLENGTH; i++) { + int si = f.applyAsInt(i); + si = partiallyWrapIndex(si, VLENGTH); + a[i] = ($bitstype$)si; + } + return a; + } + + private static boolean indicesInRange($bitstype$[] indices) { + int length = indices.length; + for ($bitstype$ si : indices) { + if (si >= ($bitstype$)length || si < ($bitstype$)(-length)) { + boolean assertsEnabled = false; + assert(assertsEnabled = true); + if (assertsEnabled) { + String msg = ("index "+si+"out of range ["+length+"] in "+ + java.util.Arrays.toString(indices)); + throw new AssertionError(msg); + } + return false; + } } - return new $shuffletype$(r); + return true; } }