diff --git a/src/hotspot/cpu/aarch64/aarch64.ad b/src/hotspot/cpu/aarch64/aarch64.ad index 4e6ed4832bf..970ddecf3bd 100644 --- a/src/hotspot/cpu/aarch64/aarch64.ad +++ b/src/hotspot/cpu/aarch64/aarch64.ad @@ -2481,7 +2481,7 @@ const RegMask* Matcher::predicate_reg_mask(void) { return &_PR_REG_mask; } -const TypeVect* Matcher::predicate_reg_type(const Type* elemTy, int length) { +const TypeVectMask* Matcher::predicate_reg_type(const Type* elemTy, int length) { return new TypeVectMask(elemTy, length); } diff --git a/src/hotspot/cpu/aarch64/aarch64_sve.ad b/src/hotspot/cpu/aarch64/aarch64_sve.ad index 6ad2d68526c..0d922d4c77a 100644 --- a/src/hotspot/cpu/aarch64/aarch64_sve.ad +++ b/src/hotspot/cpu/aarch64/aarch64_sve.ad @@ -5744,4 +5744,19 @@ instruct vmask_lasttrue_partial(iRegINoSp dst, pReg src, pReg ptmp, rFlagsReg cr __ sve_vmask_lasttrue($dst$$Register, bt, as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg)); %} ins_pipe(pipe_slow); -%} \ No newline at end of file +%} + +// ---------------------------- Vector mask generation --------------------------- +instruct vmask_gen(pRegGov pg, iRegL len, rFlagsReg cr) %{ + predicate(UseSVE > 0); + match(Set pg (VectorMaskGen len)); + effect(KILL cr); + ins_cost(SVE_COST); + format %{ "sve_whilelo $pg, zr, $len\t # sve" %} + ins_encode %{ + BasicType bt = Matcher::vector_element_basic_type(this); + Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt); + __ sve_whilelo(as_PRegister($pg$$reg), size, zr, as_Register($len$$reg)); + %} + ins_pipe(pipe_slow); +%} diff --git a/src/hotspot/cpu/aarch64/aarch64_sve_ad.m4 b/src/hotspot/cpu/aarch64/aarch64_sve_ad.m4 index 65de321a6e1..10f9096b54b 100644 --- a/src/hotspot/cpu/aarch64/aarch64_sve_ad.m4 +++ b/src/hotspot/cpu/aarch64/aarch64_sve_ad.m4 @@ -3174,4 +3174,19 @@ instruct vmask_lasttrue_partial(iRegINoSp dst, pReg src, pReg ptmp, rFlagsReg cr __ sve_vmask_lasttrue($dst$$Register, bt, as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg)); %} ins_pipe(pipe_slow); -%}dnl +%} + +// ---------------------------- Vector mask generation --------------------------- +instruct vmask_gen(pRegGov pg, iRegL len, rFlagsReg cr) %{ + predicate(UseSVE > 0); + match(Set pg (VectorMaskGen len)); + effect(KILL cr); + ins_cost(SVE_COST); + format %{ "sve_whilelo $pg, zr, $len\t # sve" %} + ins_encode %{ + BasicType bt = Matcher::vector_element_basic_type(this); + Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt); + __ sve_whilelo(as_PRegister($pg$$reg), size, zr, as_Register($len$$reg)); + %} + ins_pipe(pipe_slow); +%} diff --git a/src/hotspot/cpu/aarch64/vm_version_aarch64.cpp b/src/hotspot/cpu/aarch64/vm_version_aarch64.cpp index e08d0b6f134..4ec813df164 100644 --- a/src/hotspot/cpu/aarch64/vm_version_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/vm_version_aarch64.cpp @@ -469,6 +469,14 @@ void VM_Version::initialize() { } } + int inline_size = (UseSVE > 0 && MaxVectorSize >= 16) ? MaxVectorSize : 0; + if (FLAG_IS_DEFAULT(ArrayOperationPartialInlineSize)) { + FLAG_SET_DEFAULT(ArrayOperationPartialInlineSize, inline_size); + } else if (ArrayOperationPartialInlineSize != 0 && ArrayOperationPartialInlineSize != inline_size) { + warning("Setting ArrayOperationPartialInlineSize to %d", inline_size); + ArrayOperationPartialInlineSize = inline_size; + } + if (FLAG_IS_DEFAULT(OptoScheduling)) { OptoScheduling = true; } diff --git a/src/hotspot/cpu/arm/arm.ad b/src/hotspot/cpu/arm/arm.ad index 9e76f430054..16217afd105 100644 --- a/src/hotspot/cpu/arm/arm.ad +++ b/src/hotspot/cpu/arm/arm.ad @@ -991,7 +991,7 @@ const RegMask* Matcher::predicate_reg_mask(void) { return NULL; } -const TypeVect* Matcher::predicate_reg_type(const Type* elemTy, int length) { +const TypeVectMask* Matcher::predicate_reg_type(const Type* elemTy, int length) { return NULL; } diff --git a/src/hotspot/cpu/ppc/ppc.ad b/src/hotspot/cpu/ppc/ppc.ad index 2ced017e2f4..5a712117e82 100644 --- a/src/hotspot/cpu/ppc/ppc.ad +++ b/src/hotspot/cpu/ppc/ppc.ad @@ -2185,7 +2185,7 @@ const RegMask* Matcher::predicate_reg_mask(void) { return NULL; } -const TypeVect* Matcher::predicate_reg_type(const Type* elemTy, int length) { +const TypeVectMask* Matcher::predicate_reg_type(const Type* elemTy, int length) { return NULL; } diff --git a/src/hotspot/cpu/s390/s390.ad b/src/hotspot/cpu/s390/s390.ad index 60c27c61a51..8a6e62a809b 100644 --- a/src/hotspot/cpu/s390/s390.ad +++ b/src/hotspot/cpu/s390/s390.ad @@ -1544,7 +1544,7 @@ const RegMask* Matcher::predicate_reg_mask(void) { return NULL; } -const TypeVect* Matcher::predicate_reg_type(const Type* elemTy, int length) { +const TypeVectMask* Matcher::predicate_reg_type(const Type* elemTy, int length) { return NULL; } diff --git a/src/hotspot/cpu/x86/x86.ad b/src/hotspot/cpu/x86/x86.ad index c7e396e2637..47e898ea46b 100644 --- a/src/hotspot/cpu/x86/x86.ad +++ b/src/hotspot/cpu/x86/x86.ad @@ -2051,7 +2051,7 @@ const RegMask* Matcher::predicate_reg_mask(void) { return &_VECTMASK_REG_mask; } -const TypeVect* Matcher::predicate_reg_type(const Type* elemTy, int length) { +const TypeVectMask* Matcher::predicate_reg_type(const Type* elemTy, int length) { return new TypeVectMask(elemTy, length); } diff --git a/src/hotspot/share/opto/c2_globals.hpp b/src/hotspot/share/opto/c2_globals.hpp index b30bb4b70a4..cc6d1d762da 100644 --- a/src/hotspot/share/opto/c2_globals.hpp +++ b/src/hotspot/share/opto/c2_globals.hpp @@ -85,7 +85,7 @@ product(intx, ArrayOperationPartialInlineSize, 0, DIAGNOSTIC, \ "Partial inline size used for small array operations" \ "(e.g. copy,cmp) acceleration.") \ - range(0, 64) \ + range(0, 256) \ \ product(bool, AlignVector, true, \ "Perform vector store/load alignment in loop") \ diff --git a/src/hotspot/share/opto/library_call.cpp b/src/hotspot/share/opto/library_call.cpp index 0270b1ac01a..ea56b6e01de 100644 --- a/src/hotspot/share/opto/library_call.cpp +++ b/src/hotspot/share/opto/library_call.cpp @@ -5427,7 +5427,7 @@ bool LibraryCallKit::inline_vectorizedMismatch() { Node* obja_adr_mem = memory(C->get_alias_index(obja_adr_t)); Node* objb_adr_mem = memory(C->get_alias_index(objb_adr_t)); - Node* vmask = _gvn.transform(new VectorMaskGenNode(ConvI2X(casted_length), TypeVect::VECTMASK, elem_bt)); + Node* vmask = _gvn.transform(VectorMaskGenNode::make(ConvI2X(casted_length), elem_bt)); Node* vload_obja = _gvn.transform(new LoadVectorMaskedNode(control(), obja_adr_mem, obja_adr, obja_adr_t, vt, vmask)); Node* vload_objb = _gvn.transform(new LoadVectorMaskedNode(control(), objb_adr_mem, objb_adr, objb_adr_t, vt, vmask)); Node* result = _gvn.transform(new VectorCmpMaskedNode(vload_obja, vload_objb, vmask, TypeInt::INT)); diff --git a/src/hotspot/share/opto/macroArrayCopy.cpp b/src/hotspot/share/opto/macroArrayCopy.cpp index 68f98070e76..37f2db44269 100644 --- a/src/hotspot/share/opto/macroArrayCopy.cpp +++ b/src/hotspot/share/opto/macroArrayCopy.cpp @@ -237,7 +237,7 @@ void PhaseMacroExpand::generate_partial_inlining_block(Node** ctrl, MergeMemNode inline_block = generate_guard(ctrl, bol_le, NULL, PROB_FAIR); stub_block = *ctrl; - Node* mask_gen = new VectorMaskGenNode(casted_length, TypeVect::VECTMASK, type); + Node* mask_gen = VectorMaskGenNode::make(casted_length, type); transform_later(mask_gen); unsigned vec_size = lane_count * type2aelembytes(type); diff --git a/src/hotspot/share/opto/matcher.hpp b/src/hotspot/share/opto/matcher.hpp index 446fb259ee5..c428193ff72 100644 --- a/src/hotspot/share/opto/matcher.hpp +++ b/src/hotspot/share/opto/matcher.hpp @@ -332,7 +332,7 @@ class Matcher : public PhaseTransform { static const bool match_rule_supported_vector_masked(int opcode, int vlen, BasicType bt); static const RegMask* predicate_reg_mask(void); - static const TypeVect* predicate_reg_type(const Type* elemTy, int length); + static const TypeVectMask* predicate_reg_type(const Type* elemTy, int length); // Vector width in bytes static const int vector_width_in_bytes(BasicType bt); diff --git a/src/hotspot/share/opto/type.cpp b/src/hotspot/share/opto/type.cpp index 2cd8925002a..56e5c54ef18 100644 --- a/src/hotspot/share/opto/type.cpp +++ b/src/hotspot/share/opto/type.cpp @@ -2390,8 +2390,7 @@ const TypeVect *TypeVect::makemask(const Type* elem, uint length) { BasicType elem_bt = elem->array_element_basic_type(); if (Matcher::has_predicated_vectors() && Matcher::match_rule_supported_vector_masked(Op_VectorLoadMask, length, elem_bt)) { - const TypeVect* mtype = Matcher::predicate_reg_type(elem, length); - return (TypeVect*)(const_cast(mtype))->hashcons(); + return TypeVectMask::make(elem, length); } else { return make(elem, length); } @@ -2505,6 +2504,15 @@ const Type *TypeVectMask::xdual() const { return new TypeVectMask(element_type()->dual(), length()); } +const TypeVectMask *TypeVectMask::make(const BasicType elem_bt, uint length) { + return make(get_const_basic_type(elem_bt), length); +} + +const TypeVectMask *TypeVectMask::make(const Type* elem, uint length) { + const TypeVectMask* mtype = Matcher::predicate_reg_type(elem, length); + return (TypeVectMask*) const_cast(mtype)->hashcons(); +} + //============================================================================= // Convenience common pre-built types. const TypePtr *TypePtr::NULL_PTR; diff --git a/src/hotspot/share/opto/type.hpp b/src/hotspot/share/opto/type.hpp index 6a927ee49a7..754e0c0adc6 100644 --- a/src/hotspot/share/opto/type.hpp +++ b/src/hotspot/share/opto/type.hpp @@ -870,6 +870,8 @@ class TypeVectMask : public TypeVect { TypeVectMask(const Type* elem, uint length) : TypeVect(VectorMask, elem, length) {} virtual bool eq(const Type *t) const; virtual const Type *xdual() const; + static const TypeVectMask* make(const BasicType elem_bt, uint length); + static const TypeVectMask* make(const Type* elem, uint length); }; //------------------------------TypePtr---------------------------------------- diff --git a/src/hotspot/share/opto/vectornode.cpp b/src/hotspot/share/opto/vectornode.cpp index 3a88fb37ab0..49be2d3e7ae 100644 --- a/src/hotspot/share/opto/vectornode.cpp +++ b/src/hotspot/share/opto/vectornode.cpp @@ -801,10 +801,10 @@ Node* LoadVectorMaskedNode::Ideal(PhaseGVN* phase, bool can_reshape) { Node* mask_len = in(3)->in(1); const TypeLong* ty = phase->type(mask_len)->isa_long(); if (ty && ty->is_con()) { - BasicType mask_bt = ((VectorMaskGenNode*)in(3))->get_elem_type(); - uint load_sz = type2aelembytes(mask_bt) * ty->get_con(); - if ( load_sz == 32 || load_sz == 64) { - assert(load_sz == 32 || MaxVectorSize > 32, "Unexpected load size"); + BasicType mask_bt = Matcher::vector_element_basic_type(in(3)); + int load_sz = type2aelembytes(mask_bt) * ty->get_con(); + assert(load_sz <= MaxVectorSize, "Unexpected load size"); + if (load_sz == MaxVectorSize) { Node* ctr = in(MemNode::Control); Node* mem = in(MemNode::Memory); Node* adr = in(MemNode::Address); @@ -820,10 +820,10 @@ Node* StoreVectorMaskedNode::Ideal(PhaseGVN* phase, bool can_reshape) { Node* mask_len = in(4)->in(1); const TypeLong* ty = phase->type(mask_len)->isa_long(); if (ty && ty->is_con()) { - BasicType mask_bt = ((VectorMaskGenNode*)in(4))->get_elem_type(); - uint load_sz = type2aelembytes(mask_bt) * ty->get_con(); - if ( load_sz == 32 || load_sz == 64) { - assert(load_sz == 32 || MaxVectorSize > 32, "Unexpected store size"); + BasicType mask_bt = Matcher::vector_element_basic_type(in(4)); + int load_sz = type2aelembytes(mask_bt) * ty->get_con(); + assert(load_sz <= MaxVectorSize, "Unexpected store size"); + if (load_sz == MaxVectorSize) { Node* ctr = in(MemNode::Control); Node* mem = in(MemNode::Memory); Node* adr = in(MemNode::Address); @@ -1425,6 +1425,12 @@ Node* ShiftVNode::Identity(PhaseGVN* phase) { return this; } +Node* VectorMaskGenNode::make(Node* length, BasicType mask_bt) { + int max_vector = Matcher::max_vector_size(mask_bt); + const TypeVectMask* t_vmask = TypeVectMask::make(mask_bt, max_vector); + return new VectorMaskGenNode(length, t_vmask); +} + Node* VectorMaskOpNode::make(Node* mask, const Type* ty, int mopc) { switch(mopc) { case Op_VectorMaskTrueCount: diff --git a/src/hotspot/share/opto/vectornode.hpp b/src/hotspot/share/opto/vectornode.hpp index 95079919f78..fc424f59a3e 100644 --- a/src/hotspot/share/opto/vectornode.hpp +++ b/src/hotspot/share/opto/vectornode.hpp @@ -919,23 +919,19 @@ class VectorCmpMaskedNode : public TypeNode { virtual int Opcode() const; }; +//------------------------------VectorMaskGenNode---------------------------------- class VectorMaskGenNode : public TypeNode { public: - VectorMaskGenNode(Node* length, const Type* ty, BasicType ety): TypeNode(ty, 2), _elemType(ety) { + VectorMaskGenNode(Node* length, const Type* ty): TypeNode(ty, 2) { init_req(1, length); } virtual int Opcode() const; - BasicType get_elem_type() { return _elemType;} - virtual uint size_of() const { return sizeof(VectorMaskGenNode); } - virtual uint ideal_reg() const { - return Op_RegVectMask; - } - - private: - BasicType _elemType; + virtual uint ideal_reg() const { return Op_RegVectMask; } + static Node* make(Node* length, BasicType vmask_bt); }; +//------------------------------VectorMaskOpNode----------------------------------- class VectorMaskOpNode : public TypeNode { public: VectorMaskOpNode(Node* mask, const Type* ty, int mopc):