diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index 5f5767f836948..c5ec1c1346f99 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -255,11 +255,6 @@ static cl::opt ForceTailFoldingStyle( "Use predicated EVL instructions for tail folding. If EVL " "is unsupported, fallback to data-without-lane-mask."))); -cl::opt llvm::EnableWideActiveLaneMask( - "enable-wide-lane-mask", cl::init(false), cl::Hidden, - cl::desc("Enable use of wide lane masks when used for control flow in " - "tail-folded loops")); - static cl::opt EnableInterleavedMemAccesses( "enable-interleaved-mem-accesses", cl::init(false), cl::Hidden, cl::desc("Enable vectorization on interleaved memory accesses in a loop")); @@ -1214,15 +1209,6 @@ class LoopVectorizationCostModel { return PartialAliasMaskingStatus == AliasMaskingStatus::Enabled; } - /// Returns true if the use of wide lane masks is requested and the loop is - /// using tail-folding with a lane mask for control flow. - bool useWideActiveLaneMask() const { - if (!EnableWideActiveLaneMask) - return false; - - return getTailFoldingStyle() == TailFoldingStyle::DataAndControlFlow; - } - /// Returns true if the instructions in this block requires predication /// for any reason, e.g. because tail folding now requires a predicate /// or because the block in the original loop was predicated. @@ -3644,12 +3630,10 @@ LoopVectorizationPlanner::selectInterleaveCount(VPlan &Plan, ElementCount VF, // 3. We don't interleave if we think that we will spill registers to memory // due to the increased register pressure. - // Only interleave tail-folded loops if wide lane masks are requested, as the - // overhead of multiple instructions to calculate the predicate is likely - // not beneficial. If an epilogue is not allowed for any other reason, - // do not interleave. - if (!CM.isEpilogueAllowed() && - !(CM.preferTailFoldedLoop() && CM.useWideActiveLaneMask())) + // Do not interleave tail-folded loops, as the overhead of multiple + // instructions to calculate the predicate is likely not beneficial. + // If an epilogue is not allowed for any other reason, do not interleave. + if (!CM.isEpilogueAllowed()) return 1; if (any_of(Plan.getVectorLoopRegion()->getEntryBasicBlock()->phis(), diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h index edb24141f6073..3f2158a1e165c 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -1232,8 +1232,17 @@ class LLVM_ABI_FOR_TEST VPInstruction : public VPRecipeWithIRFlags, // Creates a mask where each lane is active (true) whilst the current // counter (first operand + index) is less than the second operand. i.e. // mask[i] = icmpt ult (op0 + i), op1 - // The size of the mask returned is VF * Multiplier (UF, third op). + // ActiveLaneMask is used for tail-folding, with the exception of the + // DataAndControlFlow style. The size of the mask returned is VF. + // When unrolled, ActiveLaneMask is duplicated. ActiveLaneMask, + // As above, but takes an additional operand (Multiplier). The size of + // the mask returned is VF * Multiplier (UF, op2). + // WideActiveLaneMask is used for control flow and is unrolled by widening, + // with one extract vector created per unroll part. + WideActiveLaneMask, + // Extracts each unrolled part of a (VF * UF) widened vector/mask. + ExtractVectorForPart, ExplicitVectorLength, // Represents the incoming loop-invariant alias-mask. All memory accesses // in the loop must stay within the active lanes. diff --git a/llvm/lib/Transforms/Vectorize/VPlanLowering.cpp b/llvm/lib/Transforms/Vectorize/VPlanLowering.cpp index 3a64567dfab0b..e3cb5a7e5c5c1 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanLowering.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanLowering.cpp @@ -117,7 +117,7 @@ void VPlanTransforms::replaceWideCanonicalIVWithWideIV( // Add a VPActiveLaneMaskPHIRecipe and related recipes to \p Plan and replace // the loop terminator with a branch-on-cond recipe with the negated -// active-lane-mask as operand. Note that this turns the loop into an +// wide-active-lane-mask as operand. Note that this turns the loop into an // uncountable one. Only the existing terminator is replaced, all other existing // recipes/users remain unchanged, except for poison-generating flags being // dropped from the canonical IV increment. Return the created @@ -127,15 +127,18 @@ void VPlanTransforms::replaceWideCanonicalIVWithWideIV( // // vector.ph: // %EntryInc = canonical-iv-increment-for-part CanonicalIVStart -// %EntryALM = active-lane-mask %EntryInc, TC +// %EntryALM = wide-active-lane-mask %EntryInc, TC +// %EntryALMPart = extract-vector-for-part %EntryALM, ir<0> // // vector.body: // ... -// %P = active-lane-mask-phi [ %EntryALM, %vector.ph ], [ %ALM, %vector.body ] +// %P = active-lane-mask-phi [ %EntryALMPart, %vector.ph ], +// [ %ALMPart, %vector.body ] // ... // %InLoopInc = canonical-iv-increment-for-part CanonicalIVIncrement -// %ALM = active-lane-mask %InLoopInc, TC -// %Negated = Not %ALM +// %ALM = wide-active-lane-mask %InLoopInc, TC +// %ALMPart = extract-vector-for-part %ALM, ir<0> +// %Negated = Not %ALMPart // branch-on-cond %Negated // static VPActiveLaneMaskPHIRecipe * @@ -147,29 +150,22 @@ addVPLaneMaskPhiAndUpdateExitBranch(VPlan &Plan) { // TODO: Check if dropping the flags is needed. TopRegion->clearCanonicalIVNUW(CanonicalIVIncrement); DebugLoc DL = CanonicalIVIncrement->getDebugLoc(); - // We can't use StartV directly in the ActiveLaneMask VPInstruction, since - // we have to take unrolling into account. Each part needs to start at - // Part * VF auto *VecPreheader = Plan.getVectorPreheader(); VPBuilder Builder(VecPreheader); - - // Create the ActiveLaneMask instruction using the correct start values. VPValue *TC = Plan.getTripCount(); - VPValue *VF = &Plan.getVF(); - - auto *EntryIncrement = - Builder.createOverflowingOp(VPInstruction::CanonicalIVIncrementForPart, - {StartV, VF}, {}, DL, "index.part.next"); - // Create the active lane mask instruction in the VPlan preheader. + // Create the wide active lane mask instruction in the VPlan preheader. VPValue *ALMMultiplier = Plan.getConstantInt(TopRegion->getCanonicalIVType(), 1); - auto *EntryALM = Builder.createNaryOp(VPInstruction::ActiveLaneMask, - {EntryIncrement, TC, ALMMultiplier}, DL, + auto *EntryALM = Builder.createNaryOp(VPInstruction::WideActiveLaneMask, + {StartV, TC, ALMMultiplier}, DL, "active.lane.mask.entry"); + EntryALM = Builder.createNaryOp(VPInstruction::ExtractVectorForPart, + {EntryALM, Plan.getConstantInt(64, 0)}, DL, + "extract.entry.alm.part"); // Now create the ActiveLaneMaskPhi recipe in the main loop using the - // preheader ActiveLaneMask instruction. + // preheader WideActiveLaneMask instruction. auto *LaneMaskPhi = new VPActiveLaneMaskPHIRecipe(EntryALM, DebugLoc::getUnknown()); auto *HeaderVPBB = TopRegion->getEntryBasicBlock(); @@ -179,12 +175,12 @@ addVPLaneMaskPhiAndUpdateExitBranch(VPlan &Plan) { // original terminator. VPRecipeBase *OriginalTerminator = EB->getTerminator(); Builder.setInsertPoint(OriginalTerminator); - auto *InLoopIncrement = Builder.createOverflowingOp( - VPInstruction::CanonicalIVIncrementForPart, - {CanonicalIVIncrement, &Plan.getVF()}, {}, DL); - auto *ALM = Builder.createNaryOp(VPInstruction::ActiveLaneMask, - {InLoopIncrement, TC, ALMMultiplier}, DL, - "active.lane.mask.next"); + auto *ALM = Builder.createNaryOp(VPInstruction::WideActiveLaneMask, + {CanonicalIVIncrement, TC, ALMMultiplier}, + DL, "active.lane.mask.next"); + ALM = Builder.createNaryOp(VPInstruction::ExtractVectorForPart, + {ALM, Plan.getConstantInt(64, 0)}, DL, + "extract.next.alm.part"); LaneMaskPhi->addBackedgeValue(ALM); // Replace the original terminator with BranchOnCond. We have to invert the @@ -214,12 +210,9 @@ void VPlanTransforms::materializeHeaderMask( VPIRFlags::WrapFlagsTy(/*HasNUW=*/true, /*HasNSW=*/false))); VPValue *Mask; if (UseActiveLaneMask) { - VPValue *ALMMultiplier = - Plan.getConstantInt(LoopRegion->getCanonicalIVType(), 1); - Mask = Builder.createNaryOp( - VPInstruction::ActiveLaneMask, - {WideCanonicalIV, Plan.getTripCount(), ALMMultiplier}, nullptr, - "active.lane.mask"); + Mask = Builder.createNaryOp(VPInstruction::ActiveLaneMask, + {WideCanonicalIV, Plan.getTripCount()}, nullptr, + "active.lane.mask"); } else { Mask = Builder.createICmp(CmpInst::ICMP_ULE, WideCanonicalIV, Plan.getOrCreateBackedgeTakenCount()); diff --git a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h index 7e7d0f9b6a280..c863d518c5464 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h +++ b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h @@ -442,6 +442,12 @@ m_ExtractLastLaneOfLastPart(const Op0_t &Op0) { return m_ExtractLastLane(m_ExtractLastPart(Op0)); } +template +inline VPInstruction_match +m_ExtractVectorForPart(const Op0_t &Op0, const Op1_t &Op1) { + return m_VPInstruction(Op0, Op1); +} + template inline VPInstruction_match m_ExtractPenultimateElement(const Op0_t &Op0) { @@ -449,9 +455,10 @@ m_ExtractPenultimateElement(const Op0_t &Op0) { } template -inline VPInstruction_match -m_ActiveLaneMask(const Op0_t &Op0, const Op1_t &Op1, const Op2_t &Op2) { - return m_VPInstruction(Op0, Op1, Op2); +inline VPInstruction_match +m_WideActiveLaneMask(const Op0_t &Op0, const Op1_t &Op1, const Op2_t &Op2) { + return m_VPInstruction(Op0, Op1, Op2); } inline VPInstruction_match m_AnyOf() { diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp index fc348c42ea7b6..b6cfe8724b88d 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp @@ -497,6 +497,7 @@ Type *llvm::computeScalarTypeForInstruction(unsigned Opcode, AssertOperandType(1, Op0Ty); return IntegerType::get(Ctx, 1); case VPInstruction::ActiveLaneMask: + case VPInstruction::WideActiveLaneMask: assert(Op0Ty->isIntegerTy() && "expected integer operand"); AssertOperandType(1, Op0Ty); return IntegerType::get(Ctx, 1); @@ -648,6 +649,7 @@ unsigned VPInstruction::getNumOperandsForOpcode() const { case Instruction::FCmp: case Instruction::ExtractElement: case Instruction::Store: + case VPInstruction::ActiveLaneMask: case VPInstruction::BranchOnCount: case VPInstruction::BranchOnTwoConds: case VPInstruction::FirstOrderRecurrenceSplice: @@ -658,10 +660,11 @@ unsigned VPInstruction::getNumOperandsForOpcode() const { case VPInstruction::WideIVStep: case VPInstruction::CalculateTripCountMinusVF: case VPInstruction::ResumeForEpilogue: + case VPInstruction::ExtractVectorForPart: return 2; case Instruction::InsertElement: case Instruction::Select: - case VPInstruction::ActiveLaneMask: + case VPInstruction::WideActiveLaneMask: case VPInstruction::ReductionStartVector: return 3; case Instruction::Call: @@ -785,20 +788,25 @@ Value *VPInstruction::generate(VPTransformState &State) { return Builder.CreateSelectFMF(Cond, Op1, Op2, getFastMathFlagsOrNone(), Name); } - case VPInstruction::ActiveLaneMask: { + case VPInstruction::ActiveLaneMask: + case VPInstruction::WideActiveLaneMask: { // Get first lane of vector induction variable. Value *VIVElem0 = State.get(getOperand(0), VPLane(0)); // Get the original loop tripcount. Value *ScalarTC = State.get(getOperand(1), VPLane(0)); + uint64_t Multiplier = + getOpcode() == VPInstruction::WideActiveLaneMask + ? cast(getOperand(2))->getZExtValue() + : 1; + // If this part of the active lane mask is scalar, generate the CMP directly // to avoid unnecessary extracts. - if (State.VF.isScalar()) + if (State.VF.isScalar() && Multiplier == 1) return Builder.CreateCmp(CmpInst::Predicate::ICMP_ULT, VIVElem0, ScalarTC, Name); - ElementCount EC = State.VF.multiplyCoefficientBy( - cast(getOperand(2))->getZExtValue()); + ElementCount EC = State.VF.multiplyCoefficientBy(Multiplier); auto *PredTy = VectorType::get(Builder.getInt1Ty(), EC); return Builder.CreateIntrinsic(Intrinsic::get_active_lane_mask, {PredTy, ScalarTC->getType()}, @@ -1104,6 +1112,17 @@ Value *VPInstruction::generate(VPTransformState &State) { return Result; } + case VPInstruction::ExtractVectorForPart: { + Value *Src = State.get(getOperand(0)); + Type *DstTy = VectorType::get(getScalarType(), State.VF); + uint64_t Part = cast(getOperand(1))->getZExtValue(); + + if (Src->getType() == DstTy) + return Src; + + return Builder.CreateExtractVector( + DstTy, Src, Builder.getInt64(State.VF.getKnownMinValue() * Part), Name); + } default: llvm_unreachable("Unsupported opcode for instruction"); } @@ -1411,9 +1430,13 @@ InstructionCost VPInstruction::computeCost(ElementCount VF, TargetTransformInfo::SK_Splice, cast(VectorTy), cast(VectorTy), {}, Ctx.CostKind, -1); } - case VPInstruction::ActiveLaneMask: { + case VPInstruction::ActiveLaneMask: + case VPInstruction::WideActiveLaneMask: { Type *ArgTy = getOperand(0)->getScalarType(); - unsigned Multiplier = cast(getOperand(2))->getZExtValue(); + uint64_t Multiplier = + getOpcode() == VPInstruction::WideActiveLaneMask + ? cast(getOperand(2))->getZExtValue() + : 1; Type *RetTy = toVectorTy(Type::getInt1Ty(Ctx.LLVMCtx), VF * Multiplier); IntrinsicCostAttributes Attrs(Intrinsic::get_active_lane_mask, RetTy, {ArgTy, ArgTy}); @@ -1622,12 +1645,14 @@ bool VPInstruction::opcodeMayReadOrWriteFromMemory() const { case VPInstruction::ExtractLastPart: case VPInstruction::ExtractPenultimateElement: case VPInstruction::ActiveLaneMask: + case VPInstruction::WideActiveLaneMask: case VPInstruction::IncomingAliasMask: case VPInstruction::ExitingIVValue: case VPInstruction::ExplicitVectorLength: case VPInstruction::FirstActiveLane: case VPInstruction::LastActiveLane: case VPInstruction::ExtractLastActive: + case VPInstruction::ExtractVectorForPart: case VPInstruction::FirstOrderRecurrenceSplice: case VPInstruction::LogicalAnd: case VPInstruction::LogicalOr: @@ -1678,6 +1703,7 @@ bool VPInstruction::usesFirstLaneOnly(const VPValue *Op) const { return vputils::onlyFirstLaneUsed(this); case Instruction::Load: case VPInstruction::ActiveLaneMask: + case VPInstruction::WideActiveLaneMask: case VPInstruction::ExplicitVectorLength: case VPInstruction::CalculateTripCountMinusVF: case VPInstruction::CanonicalIVIncrementForPart: @@ -1750,6 +1776,9 @@ void VPInstruction::printRecipe(raw_ostream &O, const Twine &Indent, case VPInstruction::ActiveLaneMask: O << "active lane mask"; break; + case VPInstruction::WideActiveLaneMask: + O << "wide active lane mask"; + break; case VPInstruction::IncomingAliasMask: O << "incoming-alias-mask"; break; @@ -1801,6 +1830,9 @@ void VPInstruction::printRecipe(raw_ostream &O, const Twine &Indent, case VPInstruction::ExtractPenultimateElement: O << "extract-penultimate-element"; break; + case VPInstruction::ExtractVectorForPart: + O << "extract-vector-for-part"; + break; case VPInstruction::ComputeReductionResult: O << "compute-reduction-result"; break; diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index b958ca253c7f4..93a38fa995747 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -2035,103 +2035,44 @@ static bool isConditionTrueViaVFAndUF(VPValue *Cond, VPlan &Plan, return SE.isKnownPredicate(CmpInst::ICMP_EQ, VectorTripCount, C); } -/// Try to replace multiple active lane masks used for control flow with -/// a single, wide active lane mask instruction followed by multiple -/// extract subvector intrinsics. This applies to the active lane mask -/// instructions both in the loop and in the preheader. -/// Incoming values of all ActiveLaneMaskPHIs are updated to use the -/// new extracts from the first active lane mask, which has it's last -/// operand (multiplier) set to UF. -static bool tryToReplaceALMWithWideALM(VPlan &Plan, ElementCount VF, - unsigned UF) { - if (!EnableWideActiveLaneMask || !VF.isVector() || UF == 1) +// Replaces ExtractVectorForPart instructions with ICMP when the VF is scalar +// and the source is a WideActiveLaneMask. The unused mask is removed later +// when removing dead recipes. +static bool replaceMaskWithCompare(VPlan &Plan, ElementCount BestVF) { + if (!BestVF.isScalar()) return false; + bool MadeChange = false; + VPBuilder Builder; VPRegionBlock *VectorRegion = Plan.getVectorLoopRegion(); + VPBasicBlock *PreheaderVPBB = Plan.getVectorPreheader(); VPBasicBlock *ExitingVPBB = VectorRegion->getExitingBasicBlock(); - auto *Term = &ExitingVPBB->back(); - using namespace llvm::VPlanPatternMatch; - if (!match(Term, m_BranchOnCond(m_Not(m_ActiveLaneMask( - m_VPValue(), m_VPValue(), m_VPValue()))))) - return false; + VPValue *Start, *TC; + uint64_t Idx; + for (VPBasicBlock *VPBB : {PreheaderVPBB, ExitingVPBB}) { + for (VPRecipeBase &R : make_early_inc_range(*VPBB)) { + if (!match(&R, m_ExtractVectorForPart( + m_WideActiveLaneMask(m_VPValue(Start), m_VPValue(TC), + m_VPValue()), + m_ConstantInt(Idx)))) + continue; - auto *Header = cast(VectorRegion->getEntry()); - LLVMContext &Ctx = Plan.getContext(); + auto *Extract = cast(&R); + Builder.setInsertPoint(Extract); - auto ExtractFromALM = [&](VPInstruction *ALM, - SmallVectorImpl &Extracts) { - DebugLoc DL = ALM->getDebugLoc(); - for (unsigned Part = 0; Part < UF; ++Part) { - SmallVector Ops; - Ops.append({ALM, Plan.getConstantInt(64, VF.getKnownMinValue() * Part)}); - auto *Ext = - new VPWidenIntrinsicRecipe(Intrinsic::vector_extract, Ops, - IntegerType::getInt1Ty(Ctx), {}, {}, DL); - Extracts[Part] = Ext; - Ext->insertAfter(ALM); - } - }; + if (Idx > 0) + Start = Builder.createAdd( + Start, Plan.getConstantInt(Start->getScalarType(), Idx)); - // Create a list of each active lane mask phi, ordered by unroll part. - SmallVector Phis(UF, nullptr); - for (VPRecipeBase &R : Header->phis()) { - auto *Phi = dyn_cast(&R); - if (!Phi) - continue; - VPValue *Index = nullptr; - match(Phi->getBackedgeValue(), - m_ActiveLaneMask(m_VPValue(Index), m_VPValue(), m_VPValue())); - assert(Index && "Expected index from ActiveLaneMask instruction"); - - uint64_t Part; - if (match(Index, - m_VPInstruction( - m_VPValue(), m_Mul(m_VPValue(), m_ConstantInt(Part))))) - Phis[Part] = Phi; - else { - // Anything other than a CanonicalIVIncrementForPart is part 0 - assert(!match( - Index, - m_VPInstruction())); - Phis[0] = Phi; + VPValue *ICmp = Builder.createICmp(CmpInst::ICMP_ULT, Start, TC); + Extract->replaceAllUsesWith(ICmp); + Extract->eraseFromParent(); + MadeChange = true; } } - assert(all_of(Phis, not_equal_to(nullptr)) && - "Expected one VPActiveLaneMaskPHIRecipe for each unroll part"); - - auto *EntryALM = cast(Phis[0]->getStartValue()); - auto *LoopALM = cast(Phis[0]->getBackedgeValue()); - - assert((EntryALM->getOpcode() == VPInstruction::ActiveLaneMask && - LoopALM->getOpcode() == VPInstruction::ActiveLaneMask) && - "Expected incoming values of Phi to be ActiveLaneMasks"); - - // When using wide lane masks, the return type of the get.active.lane.mask - // intrinsic is VF x UF (last operand). - VPValue *ALMMultiplier = Plan.getConstantInt(64, UF); - EntryALM->setOperand(2, ALMMultiplier); - LoopALM->setOperand(2, ALMMultiplier); - - // Create UF x extract vectors and insert into preheader. - SmallVector EntryExtracts(UF); - ExtractFromALM(EntryALM, EntryExtracts); - - // Create UF x extract vectors and insert before the loop compare & branch, - // updating the compare to use the first extract. - SmallVector LoopExtracts(UF); - ExtractFromALM(LoopALM, LoopExtracts); - VPInstruction *Not = cast(Term->getOperand(0)); - Not->setOperand(0, LoopExtracts[0]); - - // Update the incoming values of active lane mask phis. - for (unsigned Part = 0; Part < UF; ++Part) { - Phis[Part]->setStartValue(EntryExtracts[Part]); - Phis[Part]->setBackedgeValue(LoopExtracts[Part]); - } - - return true; + return MadeChange; } /// Try to simplify the branch condition of \p Plan. This may restrict the @@ -2149,10 +2090,13 @@ static bool simplifyBranchConditionForVFAndUF(VPlan &Plan, ElementCount BestVF, if (match(Term, m_BranchOnCount( m_CombineOr(m_CanIVInc, m_c_Add(m_CanIVInc, m_LiveIn())), m_VPValue())) || - match(Term, m_BranchOnCond(m_Not(m_ActiveLaneMask( - m_VPValue(), m_VPValue(), m_VPValue()))))) { + match(Term, + m_BranchOnCond(m_Not(m_ExtractVectorForPart( + m_WideActiveLaneMask(m_VPValue(), m_VPValue(), m_VPValue()), + m_ZeroInt()))))) { // Try to simplify the branch condition if VectorTC <= VF * UF when the - // latch terminator is BranchOnCount or BranchOnCond(Not(ActiveLaneMask)). + // latch terminator is BranchOnCount or + // BranchOnCond(Not(ExtractVectorForPart(WideActiveLaneMask), 0)) const SCEV *VectorTripCount = vputils::getSCEVExprForVPValue(&Plan.getVectorTripCount(), PSE); if (isa(VectorTripCount)) @@ -2196,7 +2140,7 @@ void VPlanTransforms::optimizeForVFAndUF(VPlan &Plan, ElementCount BestVF, assert(Plan.hasVF(BestVF) && "BestVF is not available in Plan"); assert(Plan.hasUF(BestUF) && "BestUF is not available in Plan"); - bool MadeChange = tryToReplaceALMWithWideALM(Plan, BestVF, BestUF); + bool MadeChange = replaceMaskWithCompare(Plan, BestVF); MadeChange |= simplifyBranchConditionForVFAndUF(Plan, BestVF, BestUF, PSE); MadeChange |= optimizeVectorInductionWidthForTCAndVFUF(Plan, BestVF, BestUF); @@ -3195,13 +3139,12 @@ static bool handleUncountableExitsWithSideEffects( VPBuilder MaskBuilder(HeaderVPBB, InsertIt); VPValue *FirstActive = MaskBuilder.createFirstActiveLane(*Cond); Type *IVScalarTy = IV->getScalarType(); - VPValue *ALMMultiplier = Plan.getConstantInt(IVScalarTy, 1); VPValue *Zero = Plan.getZero(IVScalarTy); FirstActive = MaskBuilder.createScalarZExtOrTrunc(FirstActive, IVScalarTy, DebugLoc()); VPValue *Mask = MaskBuilder.createNaryOp(VPInstruction::ActiveLaneMask, - {Zero, FirstActive, ALMMultiplier}, - DebugLoc(), "uncountable.exit.mask"); + {Zero, FirstActive}, DebugLoc(), + "uncountable.exit.mask"); // Convert all other memory operations to use the mask. for (VPBasicBlock *VPBB : vp_rpo_plain_cfg_loop_body(HeaderVPBB)) diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h index 4b59d37150ff4..7d2a263efce6d 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h @@ -39,7 +39,6 @@ class VPRecipeBuilder; struct VFRange; LLVM_ABI_FOR_TEST extern cl::opt VerifyEachVPlan; -LLVM_ABI_FOR_TEST extern cl::opt EnableWideActiveLaneMask; #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) LLVM_ABI_FOR_TEST extern cl::opt VPlanPrintBeforeAll; diff --git a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp index a430de94cca14..21ea2a6b74ff5 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp @@ -332,6 +332,12 @@ void UnrollState::unrollRecipeByUF(VPRecipeBase &R) { Copy->setOperand(1, getValueForPart(Op, Part)); continue; } + if (match(&R, m_VPInstruction( + m_VPValue(Op), m_VPValue()))) { + Copy->setOperand(0, Op); + Copy->setOperand(1, Plan.getConstantInt(64, Part)); + continue; + } if (isa(R)) { VPBuilder Builder(&R); const DataLayout &DL = Plan.getDataLayout(); @@ -471,6 +477,13 @@ void UnrollState::unrollBlock(VPBlockBase *VPB) { continue; } + if (match(&R, + m_WideActiveLaneMask(m_VPValue(), m_VPValue(), m_VPValue()))) { + auto *ALM = cast(&R); + ALM->setOperand(2, getConstantInt(UF)); + continue; + } + auto *SingleDef = dyn_cast(&R); if (SingleDef && vputils::isUniformAcrossVFsAndUFs(SingleDef)) { addUniformForAllParts(SingleDef); diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp index c01ddfbd24263..e356b2cf9d955 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp @@ -465,13 +465,13 @@ bool vputils::isUniformAcrossVFsAndUFs(const VPValue *V) { const VPRecipeBase *R = V->getDefiningRecipe(); const VPBasicBlock *VPBB = R ? R->getParent() : nullptr; const VPlan *Plan = VPBB ? VPBB->getPlan() : nullptr; - if (VPBB) { - if ((VPBB == Plan->getVectorPreheader() || VPBB == Plan->getEntry())) { - if (match(V->getDefiningRecipe(), - m_VPInstruction())) - return false; - return all_of(R->operands(), isUniformAcrossVFsAndUFs); - } + if (VPBB && + (VPBB == Plan->getVectorPreheader() || VPBB == Plan->getEntry())) { + if (match(R, + m_VPInstruction()) || + match(R, m_ExtractVectorForPart(m_VPValue(), m_VPValue()))) + return false; + return all_of(R->operands(), isUniformAcrossVFsAndUFs); } return TypeSwitch(R) diff --git a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp index a4d6cc0ecb480..ba4251f400df0 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp @@ -196,10 +196,8 @@ bool VPlanVerifier::verifyLastActiveLaneRecipe( // loop region's abstract header mask; afterwards it is an active lane mask // (an intrinsic or a phi), or the icmp checked below. if (match(Mask, m_HeaderMask()) || isa(Mask) || - match(Mask, m_VPInstruction())) - continue; - - if (match(Mask, m_ActiveLaneMask(m_VPValue(), m_VPValue(), m_VPValue()))) + match(Mask, m_VPInstruction()) || + match(Mask, m_VPInstruction())) continue; CmpPredicate Pred; diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/fixed-wide-lane-mask.ll b/llvm/test/Transforms/LoopVectorize/AArch64/fixed-wide-lane-mask.ll index 17cc76c52c7e3..090cb47dd315e 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/fixed-wide-lane-mask.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/fixed-wide-lane-mask.ll @@ -1,7 +1,7 @@ -; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter-out-after "^middle.block:" --version 4 -; RUN: opt -S -passes=loop-vectorize -scalable-vectorization=off -tail-folding-policy=must-fold-tail -enable-wide-lane-mask \ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "^middle.block:" --version 4 +; RUN: opt -S -passes=loop-vectorize -scalable-vectorization=off -tail-folding-policy=must-fold-tail \ ; RUN: -force-vector-width=4 -force-vector-interleave=1 < %s | FileCheck %s -check-prefix CHECK-UF1 -; RUN: opt -S --passes=loop-vectorize -scalable-vectorization=off -tail-folding-policy=must-fold-tail -enable-wide-lane-mask \ +; RUN: opt -S --passes=loop-vectorize -scalable-vectorization=off -tail-folding-policy=must-fold-tail \ ; RUN: -force-vector-width=4 -force-vector-interleave=4 < %s | FileCheck %s -check-prefix CHECK-UF4 target triple = "aarch64-unknown-linux" @@ -10,13 +10,13 @@ define void @fixed_wide_active_lane_mask(ptr noalias %dst, ptr noalias readonly ; CHECK-UF1-LABEL: define void @fixed_wide_active_lane_mask( ; CHECK-UF1-SAME: ptr noalias [[DST:%.*]], ptr noalias readonly [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] { ; CHECK-UF1-NEXT: entry: -; CHECK-UF1-NEXT: br label [[ENTRY:%.*]] +; CHECK-UF1-NEXT: br label [[VECTOR_PH:%.*]] ; CHECK-UF1: vector.ph: ; CHECK-UF1-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 0, i64 [[N]]) ; CHECK-UF1-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK-UF1: vector.body: -; CHECK-UF1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-UF1-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[ENTRY]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ] +; CHECK-UF1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] +; CHECK-UF1-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ] ; CHECK-UF1-NEXT: [[TMP3:%.*]] = load i32, ptr [[SRC]], align 4 ; CHECK-UF1-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[TMP3]], i64 0 ; CHECK-UF1-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer @@ -32,20 +32,20 @@ define void @fixed_wide_active_lane_mask(ptr noalias %dst, ptr noalias readonly ; CHECK-UF4-LABEL: define void @fixed_wide_active_lane_mask( ; CHECK-UF4-SAME: ptr noalias [[DST:%.*]], ptr noalias readonly [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] { ; CHECK-UF4-NEXT: entry: -; CHECK-UF4-NEXT: br label [[ENTRY:%.*]] +; CHECK-UF4-NEXT: br label [[VECTOR_PH:%.*]] ; CHECK-UF4: vector.ph: ; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i64(i64 0, i64 [[N]]) -; CHECK-UF4-NEXT: [[TMP4:%.*]] = call <4 x i1> @llvm.vector.extract.v4i1.v16i1(<16 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 12) -; CHECK-UF4-NEXT: [[TMP3:%.*]] = call <4 x i1> @llvm.vector.extract.v4i1.v16i1(<16 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 8) -; CHECK-UF4-NEXT: [[TMP2:%.*]] = call <4 x i1> @llvm.vector.extract.v4i1.v16i1(<16 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 4) ; CHECK-UF4-NEXT: [[TMP1:%.*]] = call <4 x i1> @llvm.vector.extract.v4i1.v16i1(<16 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 0) -; CHECK-UF4-NEXT: br label [[VECTOR_BODY1:%.*]] +; CHECK-UF4-NEXT: [[TMP2:%.*]] = call <4 x i1> @llvm.vector.extract.v4i1.v16i1(<16 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 4) +; CHECK-UF4-NEXT: [[TMP3:%.*]] = call <4 x i1> @llvm.vector.extract.v4i1.v16i1(<16 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 8) +; CHECK-UF4-NEXT: [[TMP4:%.*]] = call <4 x i1> @llvm.vector.extract.v4i1.v16i1(<16 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 12) +; CHECK-UF4-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK-UF4: vector.body: -; CHECK-UF4-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY1]] ] -; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <4 x i1> [ [[TMP1]], [[ENTRY]] ], [ [[TMP9:%.*]], [[VECTOR_BODY1]] ] -; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK4:%.*]] = phi <4 x i1> [ [[TMP2]], [[ENTRY]] ], [ [[TMP10:%.*]], [[VECTOR_BODY1]] ] -; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK5:%.*]] = phi <4 x i1> [ [[TMP3]], [[ENTRY]] ], [ [[TMP11:%.*]], [[VECTOR_BODY1]] ] -; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK6:%.*]] = phi <4 x i1> [ [[TMP4]], [[ENTRY]] ], [ [[TMP12:%.*]], [[VECTOR_BODY1]] ] +; CHECK-UF4-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] +; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <4 x i1> [ [[TMP1]], [[VECTOR_PH]] ], [ [[TMP9:%.*]], [[VECTOR_BODY]] ] +; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK4:%.*]] = phi <4 x i1> [ [[TMP2]], [[VECTOR_PH]] ], [ [[TMP10:%.*]], [[VECTOR_BODY]] ] +; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK5:%.*]] = phi <4 x i1> [ [[TMP3]], [[VECTOR_PH]] ], [ [[TMP11:%.*]], [[VECTOR_BODY]] ] +; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK6:%.*]] = phi <4 x i1> [ [[TMP4]], [[VECTOR_PH]] ], [ [[TMP12:%.*]], [[VECTOR_BODY]] ] ; CHECK-UF4-NEXT: [[TMP7:%.*]] = load i32, ptr [[SRC]], align 4 ; CHECK-UF4-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[TMP7]], i64 0 ; CHECK-UF4-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer @@ -59,13 +59,13 @@ define void @fixed_wide_active_lane_mask(ptr noalias %dst, ptr noalias readonly ; CHECK-UF4-NEXT: call void @llvm.masked.store.v4i32.p0(<4 x i32> [[BROADCAST_SPLAT]], ptr align 4 [[TMP19]], <4 x i1> [[ACTIVE_LANE_MASK6]]) ; CHECK-UF4-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 16 ; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK_NEXT:%.*]] = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i64(i64 [[INDEX_NEXT]], i64 [[N]]) -; CHECK-UF4-NEXT: [[TMP12]] = call <4 x i1> @llvm.vector.extract.v4i1.v16i1(<16 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 12) -; CHECK-UF4-NEXT: [[TMP11]] = call <4 x i1> @llvm.vector.extract.v4i1.v16i1(<16 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 8) -; CHECK-UF4-NEXT: [[TMP10]] = call <4 x i1> @llvm.vector.extract.v4i1.v16i1(<16 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 4) ; CHECK-UF4-NEXT: [[TMP9]] = call <4 x i1> @llvm.vector.extract.v4i1.v16i1(<16 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0) +; CHECK-UF4-NEXT: [[TMP10]] = call <4 x i1> @llvm.vector.extract.v4i1.v16i1(<16 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 4) +; CHECK-UF4-NEXT: [[TMP11]] = call <4 x i1> @llvm.vector.extract.v4i1.v16i1(<16 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 8) +; CHECK-UF4-NEXT: [[TMP12]] = call <4 x i1> @llvm.vector.extract.v4i1.v16i1(<16 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 12) ; CHECK-UF4-NEXT: [[TMP21:%.*]] = extractelement <4 x i1> [[TMP9]], i64 0 ; CHECK-UF4-NEXT: [[TMP20:%.*]] = xor i1 [[TMP21]], true -; CHECK-UF4-NEXT: br i1 [[TMP20]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY1]], !llvm.loop [[LOOP0:![0-9]+]] +; CHECK-UF4-NEXT: br i1 [[TMP20]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] ; CHECK-UF4: middle.block: ; entry: @@ -84,14 +84,107 @@ for.end: ret void } -attributes #0 = { nounwind "target-features"="+neon,+sve" } +define void @fixed_wide_active_lane_mask_i32_tc(ptr %src, ptr %dst, i32 %n) #0 { +; CHECK-UF1-LABEL: define void @fixed_wide_active_lane_mask_i32_tc( +; CHECK-UF1-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]], i32 [[N:%.*]]) #[[ATTR0]] { +; CHECK-UF1-NEXT: entry: +; CHECK-UF1-NEXT: [[SRC2:%.*]] = ptrtoaddr ptr [[SRC]] to i64 +; CHECK-UF1-NEXT: [[DST1:%.*]] = ptrtoaddr ptr [[DST]] to i64 +; CHECK-UF1-NEXT: br label [[VECTOR_MEMCHECK:%.*]] +; CHECK-UF1: vector.memcheck: +; CHECK-UF1-NEXT: [[TMP0:%.*]] = sub i64 [[DST1]], [[SRC2]] +; CHECK-UF1-NEXT: [[TMP1:%.*]] = sub i64 [[TMP0]], 1 +; CHECK-UF1-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP1]], 31 +; CHECK-UF1-NEXT: br i1 [[DIFF_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] +; CHECK-UF1: vector.ph: +; CHECK-UF1-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 0, i32 [[N]]) +; CHECK-UF1-NEXT: br label [[VECTOR_BODY:%.*]] +; CHECK-UF1: vector.body: +; CHECK-UF1-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] +; CHECK-UF1-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ] +; CHECK-UF1-NEXT: [[TMP2:%.*]] = getelementptr inbounds double, ptr [[SRC]], i32 [[INDEX]] +; CHECK-UF1-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x double> @llvm.masked.load.v4f64.p0(ptr align 8 [[TMP2]], <4 x i1> [[ACTIVE_LANE_MASK]], <4 x double> poison) +; CHECK-UF1-NEXT: [[TMP3:%.*]] = fmul <4 x double> [[WIDE_MASKED_LOAD]], splat (double 3.000000e+00) +; CHECK-UF1-NEXT: [[TMP4:%.*]] = getelementptr inbounds double, ptr [[DST]], i32 [[INDEX]] +; CHECK-UF1-NEXT: call void @llvm.masked.store.v4f64.p0(<4 x double> [[TMP3]], ptr align 8 [[TMP4]], <4 x i1> [[ACTIVE_LANE_MASK]]) +; CHECK-UF1-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 4 +; CHECK-UF1-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 [[INDEX_NEXT]], i32 [[N]]) +; CHECK-UF1-NEXT: [[TMP5:%.*]] = extractelement <4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0 +; CHECK-UF1-NEXT: [[TMP6:%.*]] = xor i1 [[TMP5]], true +; CHECK-UF1-NEXT: br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]] +; CHECK-UF1: middle.block: +; +; CHECK-UF4-LABEL: define void @fixed_wide_active_lane_mask_i32_tc( +; CHECK-UF4-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]], i32 [[N:%.*]]) #[[ATTR0]] { +; CHECK-UF4-NEXT: entry: +; CHECK-UF4-NEXT: [[SRC2:%.*]] = ptrtoaddr ptr [[SRC]] to i64 +; CHECK-UF4-NEXT: [[DST1:%.*]] = ptrtoaddr ptr [[DST]] to i64 +; CHECK-UF4-NEXT: br label [[VECTOR_MEMCHECK:%.*]] +; CHECK-UF4: vector.memcheck: +; CHECK-UF4-NEXT: [[TMP0:%.*]] = sub i64 [[DST1]], [[SRC2]] +; CHECK-UF4-NEXT: [[TMP1:%.*]] = sub i64 [[TMP0]], 1 +; CHECK-UF4-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP1]], 127 +; CHECK-UF4-NEXT: br i1 [[DIFF_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] +; CHECK-UF4: vector.ph: +; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32 0, i32 [[N]]) +; CHECK-UF4-NEXT: [[TMP2:%.*]] = call <4 x i1> @llvm.vector.extract.v4i1.v16i1(<16 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 0) +; CHECK-UF4-NEXT: [[TMP3:%.*]] = call <4 x i1> @llvm.vector.extract.v4i1.v16i1(<16 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 4) +; CHECK-UF4-NEXT: [[TMP4:%.*]] = call <4 x i1> @llvm.vector.extract.v4i1.v16i1(<16 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 8) +; CHECK-UF4-NEXT: [[TMP5:%.*]] = call <4 x i1> @llvm.vector.extract.v4i1.v16i1(<16 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 12) +; CHECK-UF4-NEXT: br label [[VECTOR_BODY:%.*]] +; CHECK-UF4: vector.body: +; CHECK-UF4-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] +; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <4 x i1> [ [[TMP2]], [[VECTOR_PH]] ], [ [[TMP18:%.*]], [[VECTOR_BODY]] ] +; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK3:%.*]] = phi <4 x i1> [ [[TMP3]], [[VECTOR_PH]] ], [ [[TMP19:%.*]], [[VECTOR_BODY]] ] +; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK4:%.*]] = phi <4 x i1> [ [[TMP4]], [[VECTOR_PH]] ], [ [[TMP20:%.*]], [[VECTOR_BODY]] ] +; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK5:%.*]] = phi <4 x i1> [ [[TMP5]], [[VECTOR_PH]] ], [ [[TMP21:%.*]], [[VECTOR_BODY]] ] +; CHECK-UF4-NEXT: [[TMP6:%.*]] = getelementptr inbounds double, ptr [[SRC]], i32 [[INDEX]] +; CHECK-UF4-NEXT: [[TMP7:%.*]] = getelementptr inbounds double, ptr [[TMP6]], i64 4 +; CHECK-UF4-NEXT: [[TMP8:%.*]] = getelementptr inbounds double, ptr [[TMP6]], i64 8 +; CHECK-UF4-NEXT: [[TMP9:%.*]] = getelementptr inbounds double, ptr [[TMP6]], i64 12 +; CHECK-UF4-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x double> @llvm.masked.load.v4f64.p0(ptr align 8 [[TMP6]], <4 x i1> [[ACTIVE_LANE_MASK]], <4 x double> poison) +; CHECK-UF4-NEXT: [[WIDE_MASKED_LOAD6:%.*]] = call <4 x double> @llvm.masked.load.v4f64.p0(ptr align 8 [[TMP7]], <4 x i1> [[ACTIVE_LANE_MASK3]], <4 x double> poison) +; CHECK-UF4-NEXT: [[WIDE_MASKED_LOAD7:%.*]] = call <4 x double> @llvm.masked.load.v4f64.p0(ptr align 8 [[TMP8]], <4 x i1> [[ACTIVE_LANE_MASK4]], <4 x double> poison) +; CHECK-UF4-NEXT: [[WIDE_MASKED_LOAD8:%.*]] = call <4 x double> @llvm.masked.load.v4f64.p0(ptr align 8 [[TMP9]], <4 x i1> [[ACTIVE_LANE_MASK5]], <4 x double> poison) +; CHECK-UF4-NEXT: [[TMP10:%.*]] = fmul <4 x double> [[WIDE_MASKED_LOAD]], splat (double 3.000000e+00) +; CHECK-UF4-NEXT: [[TMP11:%.*]] = fmul <4 x double> [[WIDE_MASKED_LOAD6]], splat (double 3.000000e+00) +; CHECK-UF4-NEXT: [[TMP12:%.*]] = fmul <4 x double> [[WIDE_MASKED_LOAD7]], splat (double 3.000000e+00) +; CHECK-UF4-NEXT: [[TMP13:%.*]] = fmul <4 x double> [[WIDE_MASKED_LOAD8]], splat (double 3.000000e+00) +; CHECK-UF4-NEXT: [[TMP14:%.*]] = getelementptr inbounds double, ptr [[DST]], i32 [[INDEX]] +; CHECK-UF4-NEXT: [[TMP15:%.*]] = getelementptr inbounds double, ptr [[TMP14]], i64 4 +; CHECK-UF4-NEXT: [[TMP16:%.*]] = getelementptr inbounds double, ptr [[TMP14]], i64 8 +; CHECK-UF4-NEXT: [[TMP17:%.*]] = getelementptr inbounds double, ptr [[TMP14]], i64 12 +; CHECK-UF4-NEXT: call void @llvm.masked.store.v4f64.p0(<4 x double> [[TMP10]], ptr align 8 [[TMP14]], <4 x i1> [[ACTIVE_LANE_MASK]]) +; CHECK-UF4-NEXT: call void @llvm.masked.store.v4f64.p0(<4 x double> [[TMP11]], ptr align 8 [[TMP15]], <4 x i1> [[ACTIVE_LANE_MASK3]]) +; CHECK-UF4-NEXT: call void @llvm.masked.store.v4f64.p0(<4 x double> [[TMP12]], ptr align 8 [[TMP16]], <4 x i1> [[ACTIVE_LANE_MASK4]]) +; CHECK-UF4-NEXT: call void @llvm.masked.store.v4f64.p0(<4 x double> [[TMP13]], ptr align 8 [[TMP17]], <4 x i1> [[ACTIVE_LANE_MASK5]]) +; CHECK-UF4-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 16 +; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK_NEXT:%.*]] = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32 [[INDEX_NEXT]], i32 [[N]]) +; CHECK-UF4-NEXT: [[TMP18]] = call <4 x i1> @llvm.vector.extract.v4i1.v16i1(<16 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0) +; CHECK-UF4-NEXT: [[TMP19]] = call <4 x i1> @llvm.vector.extract.v4i1.v16i1(<16 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 4) +; CHECK-UF4-NEXT: [[TMP20]] = call <4 x i1> @llvm.vector.extract.v4i1.v16i1(<16 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 8) +; CHECK-UF4-NEXT: [[TMP21]] = call <4 x i1> @llvm.vector.extract.v4i1.v16i1(<16 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 12) +; CHECK-UF4-NEXT: [[TMP22:%.*]] = extractelement <4 x i1> [[TMP18]], i64 0 +; CHECK-UF4-NEXT: [[TMP23:%.*]] = xor i1 [[TMP22]], true +; CHECK-UF4-NEXT: br i1 [[TMP23]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]] +; CHECK-UF4: middle.block: +; +entry: + br label %for.body -;. -; CHECK-UF1: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]} -; CHECK-UF1: [[META1]] = !{!"llvm.loop.isvectorized", i32 1} -; CHECK-UF1: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"} -;. -; CHECK-UF4: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]} -; CHECK-UF4: [[META1]] = !{!"llvm.loop.isvectorized", i32 1} -; CHECK-UF4: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"} -;. +for.body: + %iv = phi i32 [ 0, %entry ], [ %iv.next, %for.body ] + %arrayidx1 = getelementptr inbounds double, ptr %src, i32 %iv + %ld = load double, ptr %arrayidx1 + %mul = fmul double %ld, 3.000000e+00 + %arrayidx2 = getelementptr inbounds double, ptr %dst, i32 %iv + store double %mul, ptr %arrayidx2 + %iv.next = add nuw nsw i32 %iv, 1 + %exitcond.not = icmp eq i32 %iv.next, %n + br i1 %exitcond.not, label %for.end, label %for.body + +for.end: + ret void +} + +attributes #0 = { nounwind "target-features"="+neon,+sve" } diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/masked-call-scalarize.ll b/llvm/test/Transforms/LoopVectorize/AArch64/masked-call-scalarize.ll index d7c4641429480..9292b8bb2c43d 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/masked-call-scalarize.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/masked-call-scalarize.ll @@ -101,13 +101,14 @@ define void @test_widen_exp_v2(ptr noalias %p2, ptr noalias %p, i64 %n) #5 { ; TFA_INTERLEAVE-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1 ; TFA_INTERLEAVE-NEXT: br label %[[VECTOR_PH:.*]] ; TFA_INTERLEAVE: [[VECTOR_PH]]: -; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i64(i64 0, i64 [[TMP0]]) -; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY1:%.*]] = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i64(i64 2, i64 [[TMP0]]) +; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 0, i64 [[TMP0]]) +; TFA_INTERLEAVE-NEXT: [[TMP13:%.*]] = call <2 x i1> @llvm.vector.extract.v2i1.v4i1(<4 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 0) +; TFA_INTERLEAVE-NEXT: [[TMP17:%.*]] = call <2 x i1> @llvm.vector.extract.v2i1.v4i1(<4 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 2) ; TFA_INTERLEAVE-NEXT: br label %[[VECTOR_BODY:.*]] ; TFA_INTERLEAVE: [[VECTOR_BODY]]: ; TFA_INTERLEAVE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE8:.*]] ] -; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[PRED_STORE_CONTINUE8]] ] -; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK2:%.*]] = phi <2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY1]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT9:%.*]], %[[PRED_STORE_CONTINUE8]] ] +; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <2 x i1> [ [[TMP13]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[PRED_STORE_CONTINUE8]] ] +; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK2:%.*]] = phi <2 x i1> [ [[TMP17]], %[[VECTOR_PH]] ], [ [[TMP16:%.*]], %[[PRED_STORE_CONTINUE8]] ] ; TFA_INTERLEAVE-NEXT: [[TMP1:%.*]] = load double, ptr [[P2]], align 8 ; TFA_INTERLEAVE-NEXT: [[TMP2:%.*]] = tail call double @llvm.exp.f64(double [[TMP1]]) #[[ATTR3:[0-9]+]] ; TFA_INTERLEAVE-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x double> poison, double [[TMP2]], i64 0 @@ -144,9 +145,9 @@ define void @test_widen_exp_v2(ptr noalias %p2, ptr noalias %p, i64 %n) #5 { ; TFA_INTERLEAVE-NEXT: br label %[[PRED_STORE_CONTINUE8]] ; TFA_INTERLEAVE: [[PRED_STORE_CONTINUE8]]: ; TFA_INTERLEAVE-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4 -; TFA_INTERLEAVE-NEXT: [[TMP13:%.*]] = add i64 [[INDEX_NEXT]], 2 -; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i64(i64 [[INDEX_NEXT]], i64 [[TMP0]]) -; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT9]] = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i64(i64 [[TMP13]], i64 [[TMP0]]) +; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT1:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 [[INDEX_NEXT]], i64 [[TMP0]]) +; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <2 x i1> @llvm.vector.extract.v2i1.v4i1(<4 x i1> [[ACTIVE_LANE_MASK_NEXT1]], i64 0) +; TFA_INTERLEAVE-NEXT: [[TMP16]] = call <2 x i1> @llvm.vector.extract.v2i1.v4i1(<4 x i1> [[ACTIVE_LANE_MASK_NEXT1]], i64 2) ; TFA_INTERLEAVE-NEXT: [[TMP14:%.*]] = extractelement <2 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0 ; TFA_INTERLEAVE-NEXT: [[TMP15:%.*]] = xor i1 [[TMP14]], true ; TFA_INTERLEAVE-NEXT: br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/masked-call.ll b/llvm/test/Transforms/LoopVectorize/AArch64/masked-call.ll index 018c280b693d8..890d0b0987240 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/masked-call.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/masked-call.ll @@ -82,13 +82,14 @@ define void @test_widen(ptr noalias %a, ptr readnone %b) #4 { ; TFA_INTERLEAVE-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64() ; TFA_INTERLEAVE-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1 ; TFA_INTERLEAVE-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 1 -; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 1025) -; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY1:%.*]] = call @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP1]], i64 1025) +; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 1025) +; TFA_INTERLEAVE-NEXT: [[TMP9:%.*]] = call @llvm.vector.extract.nxv2i1.nxv4i1( [[ACTIVE_LANE_MASK_ENTRY]], i64 0) +; TFA_INTERLEAVE-NEXT: [[TMP13:%.*]] = call @llvm.vector.extract.nxv2i1.nxv4i1( [[ACTIVE_LANE_MASK_ENTRY]], i64 2) ; TFA_INTERLEAVE-NEXT: br label %[[VECTOR_BODY:.*]] ; TFA_INTERLEAVE: [[VECTOR_BODY]]: ; TFA_INTERLEAVE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] -; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ] -; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK2:%.*]] = phi [ [[ACTIVE_LANE_MASK_ENTRY1]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT4:%.*]], %[[VECTOR_BODY]] ] +; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi [ [[TMP9]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ] +; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK2:%.*]] = phi [ [[TMP13]], %[[VECTOR_PH]] ], [ [[TMP12:%.*]], %[[VECTOR_BODY]] ] ; TFA_INTERLEAVE-NEXT: [[TMP3:%.*]] = getelementptr i64, ptr [[B]], i64 [[INDEX]] ; TFA_INTERLEAVE-NEXT: [[TMP4:%.*]] = getelementptr i64, ptr [[TMP3]], i64 [[TMP1]] ; TFA_INTERLEAVE-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call @llvm.masked.load.nxv2i64.p0(ptr align 8 [[TMP3]], [[ACTIVE_LANE_MASK]], poison) @@ -100,9 +101,9 @@ define void @test_widen(ptr noalias %a, ptr readnone %b) #4 { ; TFA_INTERLEAVE-NEXT: call void @llvm.masked.store.nxv2i64.p0( [[TMP5]], ptr align 8 [[TMP7]], [[ACTIVE_LANE_MASK]]) ; TFA_INTERLEAVE-NEXT: call void @llvm.masked.store.nxv2i64.p0( [[TMP6]], ptr align 8 [[TMP8]], [[ACTIVE_LANE_MASK2]]) ; TFA_INTERLEAVE-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP2]] -; TFA_INTERLEAVE-NEXT: [[TMP9:%.*]] = add i64 [[INDEX_NEXT]], [[TMP1]] -; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX_NEXT]], i64 1025) -; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT4]] = call @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP9]], i64 1025) +; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT1:%.*]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT]], i64 1025) +; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.vector.extract.nxv2i1.nxv4i1( [[ACTIVE_LANE_MASK_NEXT1]], i64 0) +; TFA_INTERLEAVE-NEXT: [[TMP12]] = call @llvm.vector.extract.nxv2i1.nxv4i1( [[ACTIVE_LANE_MASK_NEXT1]], i64 2) ; TFA_INTERLEAVE-NEXT: [[TMP10:%.*]] = extractelement [[ACTIVE_LANE_MASK_NEXT]], i64 0 ; TFA_INTERLEAVE-NEXT: [[TMP11:%.*]] = xor i1 [[TMP10]], true ; TFA_INTERLEAVE-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] @@ -215,13 +216,14 @@ define void @test_if_then(ptr noalias %a, ptr readnone %b) #4 { ; TFA_INTERLEAVE-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64() ; TFA_INTERLEAVE-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1 ; TFA_INTERLEAVE-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 1 -; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 1025) -; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY1:%.*]] = call @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP1]], i64 1025) +; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 1025) +; TFA_INTERLEAVE-NEXT: [[TMP13:%.*]] = call @llvm.vector.extract.nxv2i1.nxv4i1( [[ACTIVE_LANE_MASK_ENTRY]], i64 0) +; TFA_INTERLEAVE-NEXT: [[TMP17:%.*]] = call @llvm.vector.extract.nxv2i1.nxv4i1( [[ACTIVE_LANE_MASK_ENTRY]], i64 2) ; TFA_INTERLEAVE-NEXT: br label %[[VECTOR_BODY:.*]] ; TFA_INTERLEAVE: [[VECTOR_BODY]]: ; TFA_INTERLEAVE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] -; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ] -; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK2:%.*]] = phi [ [[ACTIVE_LANE_MASK_ENTRY1]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT5:%.*]], %[[VECTOR_BODY]] ] +; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi [ [[TMP13]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ] +; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK2:%.*]] = phi [ [[TMP17]], %[[VECTOR_PH]] ], [ [[TMP16:%.*]], %[[VECTOR_BODY]] ] ; TFA_INTERLEAVE-NEXT: [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]] ; TFA_INTERLEAVE-NEXT: [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[TMP3]], i64 [[TMP1]] ; TFA_INTERLEAVE-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call @llvm.masked.load.nxv2i64.p0(ptr align 8 [[TMP3]], [[ACTIVE_LANE_MASK]], poison) @@ -239,9 +241,9 @@ define void @test_if_then(ptr noalias %a, ptr readnone %b) #4 { ; TFA_INTERLEAVE-NEXT: call void @llvm.masked.store.nxv2i64.p0( [[PREDPHI]], ptr align 8 [[TMP11]], [[ACTIVE_LANE_MASK]]) ; TFA_INTERLEAVE-NEXT: call void @llvm.masked.store.nxv2i64.p0( [[PREDPHI4]], ptr align 8 [[TMP12]], [[ACTIVE_LANE_MASK2]]) ; TFA_INTERLEAVE-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP2]] -; TFA_INTERLEAVE-NEXT: [[TMP13:%.*]] = add i64 [[INDEX_NEXT]], [[TMP1]] -; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX_NEXT]], i64 1025) -; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT5]] = call @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP13]], i64 1025) +; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT1:%.*]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT]], i64 1025) +; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.vector.extract.nxv2i1.nxv4i1( [[ACTIVE_LANE_MASK_NEXT1]], i64 0) +; TFA_INTERLEAVE-NEXT: [[TMP16]] = call @llvm.vector.extract.nxv2i1.nxv4i1( [[ACTIVE_LANE_MASK_NEXT1]], i64 2) ; TFA_INTERLEAVE-NEXT: [[TMP14:%.*]] = extractelement [[ACTIVE_LANE_MASK_NEXT]], i64 0 ; TFA_INTERLEAVE-NEXT: [[TMP15:%.*]] = xor i1 [[TMP14]], true ; TFA_INTERLEAVE-NEXT: br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]] @@ -373,13 +375,14 @@ define void @test_widen_if_then_else(ptr noalias %a, ptr readnone %b) #4 { ; TFA_INTERLEAVE-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64() ; TFA_INTERLEAVE-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1 ; TFA_INTERLEAVE-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 1 -; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 1025) -; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY1:%.*]] = call @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP1]], i64 1025) +; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 1025) +; TFA_INTERLEAVE-NEXT: [[TMP19:%.*]] = call @llvm.vector.extract.nxv2i1.nxv4i1( [[ACTIVE_LANE_MASK_ENTRY]], i64 0) +; TFA_INTERLEAVE-NEXT: [[TMP23:%.*]] = call @llvm.vector.extract.nxv2i1.nxv4i1( [[ACTIVE_LANE_MASK_ENTRY]], i64 2) ; TFA_INTERLEAVE-NEXT: br label %[[VECTOR_BODY:.*]] ; TFA_INTERLEAVE: [[VECTOR_BODY]]: ; TFA_INTERLEAVE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] -; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ] -; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK2:%.*]] = phi [ [[ACTIVE_LANE_MASK_ENTRY1]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT5:%.*]], %[[VECTOR_BODY]] ] +; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi [ [[TMP19]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ] +; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK2:%.*]] = phi [ [[TMP23]], %[[VECTOR_PH]] ], [ [[TMP22:%.*]], %[[VECTOR_BODY]] ] ; TFA_INTERLEAVE-NEXT: [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]] ; TFA_INTERLEAVE-NEXT: [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[TMP3]], i64 [[TMP1]] ; TFA_INTERLEAVE-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call @llvm.masked.load.nxv2i64.p0(ptr align 8 [[TMP3]], [[ACTIVE_LANE_MASK]], poison) @@ -403,9 +406,9 @@ define void @test_widen_if_then_else(ptr noalias %a, ptr readnone %b) #4 { ; TFA_INTERLEAVE-NEXT: call void @llvm.masked.store.nxv2i64.p0( [[PREDPHI]], ptr align 8 [[TMP17]], [[ACTIVE_LANE_MASK]]) ; TFA_INTERLEAVE-NEXT: call void @llvm.masked.store.nxv2i64.p0( [[PREDPHI4]], ptr align 8 [[TMP18]], [[ACTIVE_LANE_MASK2]]) ; TFA_INTERLEAVE-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP2]] -; TFA_INTERLEAVE-NEXT: [[TMP19:%.*]] = add i64 [[INDEX_NEXT]], [[TMP1]] -; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX_NEXT]], i64 1025) -; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT5]] = call @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP19]], i64 1025) +; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT1:%.*]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT]], i64 1025) +; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.vector.extract.nxv2i1.nxv4i1( [[ACTIVE_LANE_MASK_NEXT1]], i64 0) +; TFA_INTERLEAVE-NEXT: [[TMP22]] = call @llvm.vector.extract.nxv2i1.nxv4i1( [[ACTIVE_LANE_MASK_NEXT1]], i64 2) ; TFA_INTERLEAVE-NEXT: [[TMP20:%.*]] = extractelement [[ACTIVE_LANE_MASK_NEXT]], i64 0 ; TFA_INTERLEAVE-NEXT: [[TMP21:%.*]] = xor i1 [[TMP20]], true ; TFA_INTERLEAVE-NEXT: br i1 [[TMP21]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]] @@ -680,13 +683,14 @@ define void @test_widen_optmask(ptr noalias %a, ptr readnone %b) #4 { ; TFA_INTERLEAVE-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64() ; TFA_INTERLEAVE-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1 ; TFA_INTERLEAVE-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 1 -; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 1025) -; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY1:%.*]] = call @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP1]], i64 1025) +; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 1025) +; TFA_INTERLEAVE-NEXT: [[TMP9:%.*]] = call @llvm.vector.extract.nxv2i1.nxv4i1( [[ACTIVE_LANE_MASK_ENTRY]], i64 0) +; TFA_INTERLEAVE-NEXT: [[TMP13:%.*]] = call @llvm.vector.extract.nxv2i1.nxv4i1( [[ACTIVE_LANE_MASK_ENTRY]], i64 2) ; TFA_INTERLEAVE-NEXT: br label %[[VECTOR_BODY:.*]] ; TFA_INTERLEAVE: [[VECTOR_BODY]]: ; TFA_INTERLEAVE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] -; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ] -; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK2:%.*]] = phi [ [[ACTIVE_LANE_MASK_ENTRY1]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT4:%.*]], %[[VECTOR_BODY]] ] +; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi [ [[TMP9]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ] +; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK2:%.*]] = phi [ [[TMP13]], %[[VECTOR_PH]] ], [ [[TMP12:%.*]], %[[VECTOR_BODY]] ] ; TFA_INTERLEAVE-NEXT: [[TMP3:%.*]] = getelementptr i64, ptr [[B]], i64 [[INDEX]] ; TFA_INTERLEAVE-NEXT: [[TMP4:%.*]] = getelementptr i64, ptr [[TMP3]], i64 [[TMP1]] ; TFA_INTERLEAVE-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call @llvm.masked.load.nxv2i64.p0(ptr align 8 [[TMP3]], [[ACTIVE_LANE_MASK]], poison) @@ -698,9 +702,9 @@ define void @test_widen_optmask(ptr noalias %a, ptr readnone %b) #4 { ; TFA_INTERLEAVE-NEXT: call void @llvm.masked.store.nxv2i64.p0( [[TMP5]], ptr align 8 [[TMP7]], [[ACTIVE_LANE_MASK]]) ; TFA_INTERLEAVE-NEXT: call void @llvm.masked.store.nxv2i64.p0( [[TMP6]], ptr align 8 [[TMP8]], [[ACTIVE_LANE_MASK2]]) ; TFA_INTERLEAVE-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP2]] -; TFA_INTERLEAVE-NEXT: [[TMP9:%.*]] = add i64 [[INDEX_NEXT]], [[TMP1]] -; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX_NEXT]], i64 1025) -; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT4]] = call @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP9]], i64 1025) +; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT1:%.*]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT]], i64 1025) +; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.vector.extract.nxv2i1.nxv4i1( [[ACTIVE_LANE_MASK_NEXT1]], i64 0) +; TFA_INTERLEAVE-NEXT: [[TMP12]] = call @llvm.vector.extract.nxv2i1.nxv4i1( [[ACTIVE_LANE_MASK_NEXT1]], i64 2) ; TFA_INTERLEAVE-NEXT: [[TMP10:%.*]] = extractelement [[ACTIVE_LANE_MASK_NEXT]], i64 0 ; TFA_INTERLEAVE-NEXT: [[TMP11:%.*]] = xor i1 [[TMP10]], true ; TFA_INTERLEAVE-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]] @@ -855,15 +859,16 @@ define double @test_widen_fmuladd_and_call(ptr noalias %a, ptr readnone %b, doub ; TFA_INTERLEAVE-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64() ; TFA_INTERLEAVE-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1 ; TFA_INTERLEAVE-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 1 -; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 1025) -; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY1:%.*]] = call @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP1]], i64 1025) +; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 1025) +; TFA_INTERLEAVE-NEXT: [[TMP17:%.*]] = call @llvm.vector.extract.nxv2i1.nxv4i1( [[ACTIVE_LANE_MASK_ENTRY]], i64 0) +; TFA_INTERLEAVE-NEXT: [[TMP21:%.*]] = call @llvm.vector.extract.nxv2i1.nxv4i1( [[ACTIVE_LANE_MASK_ENTRY]], i64 2) ; TFA_INTERLEAVE-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement poison, double [[M]], i64 0 ; TFA_INTERLEAVE-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector [[BROADCAST_SPLATINSERT]], poison, zeroinitializer ; TFA_INTERLEAVE-NEXT: br label %[[VECTOR_BODY:.*]] ; TFA_INTERLEAVE: [[VECTOR_BODY]]: ; TFA_INTERLEAVE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] -; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ] -; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK2:%.*]] = phi [ [[ACTIVE_LANE_MASK_ENTRY1]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT4:%.*]], %[[VECTOR_BODY]] ] +; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi [ [[TMP17]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ] +; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK2:%.*]] = phi [ [[TMP21]], %[[VECTOR_PH]] ], [ [[TMP20:%.*]], %[[VECTOR_BODY]] ] ; TFA_INTERLEAVE-NEXT: [[VEC_PHI:%.*]] = phi double [ 0.000000e+00, %[[VECTOR_PH]] ], [ [[TMP16:%.*]], %[[VECTOR_BODY]] ] ; TFA_INTERLEAVE-NEXT: [[TMP3:%.*]] = getelementptr double, ptr [[B]], i64 [[INDEX]] ; TFA_INTERLEAVE-NEXT: [[TMP4:%.*]] = getelementptr double, ptr [[TMP3]], i64 [[TMP1]] @@ -884,9 +889,9 @@ define double @test_widen_fmuladd_and_call(ptr noalias %a, ptr readnone %b, doub ; TFA_INTERLEAVE-NEXT: [[TMP15:%.*]] = select [[ACTIVE_LANE_MASK2]], [[TMP6]], splat (double -0.000000e+00) ; TFA_INTERLEAVE-NEXT: [[TMP16]] = call double @llvm.vector.reduce.fadd.nxv2f64(double [[TMP14]], [[TMP15]]) ; TFA_INTERLEAVE-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP2]] -; TFA_INTERLEAVE-NEXT: [[TMP17:%.*]] = add i64 [[INDEX_NEXT]], [[TMP1]] -; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX_NEXT]], i64 1025) -; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT4]] = call @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP17]], i64 1025) +; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT1:%.*]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT]], i64 1025) +; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.vector.extract.nxv2i1.nxv4i1( [[ACTIVE_LANE_MASK_NEXT1]], i64 0) +; TFA_INTERLEAVE-NEXT: [[TMP20]] = call @llvm.vector.extract.nxv2i1.nxv4i1( [[ACTIVE_LANE_MASK_NEXT1]], i64 2) ; TFA_INTERLEAVE-NEXT: [[TMP18:%.*]] = extractelement [[ACTIVE_LANE_MASK_NEXT]], i64 0 ; TFA_INTERLEAVE-NEXT: [[TMP19:%.*]] = xor i1 [[TMP18]], true ; TFA_INTERLEAVE-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]] @@ -1013,8 +1018,8 @@ define void @test_widen_exp_v2(ptr noalias %p2, ptr noalias %p, i64 %n) #5 { ; TFA_INTERLEAVE-NEXT: br label %[[PRED_STORE_CONTINUE4]] ; TFA_INTERLEAVE: [[PRED_STORE_CONTINUE4]]: ; TFA_INTERLEAVE-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 2 -; TFA_INTERLEAVE-NEXT: [[TMP4:%.*]] = add i64 [[INDEX_NEXT]], 1 ; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = icmp ult i64 [[INDEX_NEXT]], [[TMP0]] +; TFA_INTERLEAVE-NEXT: [[TMP4:%.*]] = add i64 [[INDEX_NEXT]], 1 ; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT5]] = icmp ult i64 [[TMP4]], [[TMP0]] ; TFA_INTERLEAVE-NEXT: [[TMP5:%.*]] = xor i1 [[ACTIVE_LANE_MASK_NEXT]], true ; TFA_INTERLEAVE-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]] diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/pr73894.ll b/llvm/test/Transforms/LoopVectorize/AArch64/pr73894.ll index 3e61a933d0ecb..ce49f94a4d43d 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/pr73894.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/pr73894.ll @@ -12,12 +12,11 @@ define i32 @pr70988(ptr %src, i32 %n) { ; CHECK-NEXT: [[UMAX:%.*]] = zext i32 [[TMP2]] to i64 ; CHECK-NEXT: br label [[VECTOR_PH:%.*]] ; CHECK: vector.ph: -; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = icmp ult i64 0, [[UMAX]] ; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY1:%.*]] = icmp ult i64 1, [[UMAX]] ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_LOAD_CONTINUE5:%.*]] ] -; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi i1 [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[PRED_LOAD_CONTINUE5]] ] +; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi i1 [ true, [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[PRED_LOAD_CONTINUE5]] ] ; CHECK-NEXT: [[ACTIVE_LANE_MASK2:%.*]] = phi i1 [ [[ACTIVE_LANE_MASK_ENTRY1]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT7:%.*]], [[PRED_LOAD_CONTINUE5]] ] ; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP17:%.*]], [[PRED_LOAD_CONTINUE5]] ] ; CHECK-NEXT: [[VEC_PHI3:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP18:%.*]], [[PRED_LOAD_CONTINUE5]] ] @@ -30,21 +29,21 @@ define i32 @pr70988(ptr %src, i32 %n) { ; CHECK: pred.load.continue: ; CHECK-NEXT: [[TMP8:%.*]] = phi i32 [ poison, [[VECTOR_BODY]] ], [ [[TMP6]], [[PRED_LOAD_IF]] ] ; CHECK-NEXT: br i1 [[ACTIVE_LANE_MASK2]], label [[PRED_LOAD_IF4:%.*]], label [[PRED_LOAD_CONTINUE5]] -; CHECK: pred.load.if4: +; CHECK: pred.load.if3: ; CHECK-NEXT: [[TMP9:%.*]] = add i64 [[INDEX]], 1 ; CHECK-NEXT: [[TMP10:%.*]] = getelementptr i32, ptr [[SRC]], i64 [[TMP9]] ; CHECK-NEXT: [[TMP11:%.*]] = load ptr, ptr [[TMP10]], align 8 ; CHECK-NEXT: [[TMP12:%.*]] = load i32, ptr [[TMP11]], align 4 ; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE5]] -; CHECK: pred.load.continue5: +; CHECK: pred.load.continue4: ; CHECK-NEXT: [[TMP14:%.*]] = phi i32 [ poison, [[PRED_LOAD_CONTINUE]] ], [ [[TMP12]], [[PRED_LOAD_IF4]] ] ; CHECK-NEXT: [[TMP15:%.*]] = tail call i32 @llvm.smax.i32(i32 [[TMP8]], i32 [[VEC_PHI]]) ; CHECK-NEXT: [[TMP16:%.*]] = tail call i32 @llvm.smax.i32(i32 [[TMP14]], i32 [[VEC_PHI3]]) ; CHECK-NEXT: [[TMP17]] = select i1 [[ACTIVE_LANE_MASK]], i32 [[TMP15]], i32 [[VEC_PHI]] ; CHECK-NEXT: [[TMP18]] = select i1 [[ACTIVE_LANE_MASK2]], i32 [[TMP16]], i32 [[VEC_PHI3]] ; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 2 -; CHECK-NEXT: [[TMP19:%.*]] = add i64 [[INDEX_NEXT]], 1 ; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = icmp ult i64 [[INDEX_NEXT]], [[UMAX]] +; CHECK-NEXT: [[TMP19:%.*]] = add i64 [[INDEX_NEXT]], 1 ; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT7]] = icmp ult i64 [[TMP19]], [[UMAX]] ; CHECK-NEXT: [[TMP20:%.*]] = xor i1 [[ACTIVE_LANE_MASK_NEXT]], true ; CHECK-NEXT: br i1 [[TMP20]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/scalable-strict-fadd.ll b/llvm/test/Transforms/LoopVectorize/AArch64/scalable-strict-fadd.ll index c79684470e794..f2a86a731a35b 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/scalable-strict-fadd.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/scalable-strict-fadd.ll @@ -300,21 +300,22 @@ define float @fadd_strict_unroll(ptr noalias nocapture readonly %a, i64 %n) #0 { ; CHECK-ORDERED-TF-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64() ; CHECK-ORDERED-TF-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3 ; CHECK-ORDERED-TF-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 2 -; CHECK-ORDERED-TF-NEXT: [[TMP3:%.*]] = shl i64 [[TMP1]], 1 -; CHECK-ORDERED-TF-NEXT: [[TMP4:%.*]] = mul i64 [[TMP1]], 3 -; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 [[N]]) -; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_ENTRY1:%.*]] = call @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP1]], i64 [[N]]) -; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_ENTRY2:%.*]] = call @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP3]], i64 [[N]]) -; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_ENTRY3:%.*]] = call @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP4]], i64 [[N]]) +; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call @llvm.get.active.lane.mask.nxv32i1.i64(i64 0, i64 [[N]]) +; CHECK-ORDERED-TF-NEXT: [[TMP17:%.*]] = call @llvm.vector.extract.nxv8i1.nxv32i1( [[ACTIVE_LANE_MASK_ENTRY]], i64 0) +; CHECK-ORDERED-TF-NEXT: [[TMP18:%.*]] = call @llvm.vector.extract.nxv8i1.nxv32i1( [[ACTIVE_LANE_MASK_ENTRY]], i64 8) +; CHECK-ORDERED-TF-NEXT: [[TMP19:%.*]] = call @llvm.vector.extract.nxv8i1.nxv32i1( [[ACTIVE_LANE_MASK_ENTRY]], i64 16) +; CHECK-ORDERED-TF-NEXT: [[TMP25:%.*]] = call @llvm.vector.extract.nxv8i1.nxv32i1( [[ACTIVE_LANE_MASK_ENTRY]], i64 24) ; CHECK-ORDERED-TF-NEXT: br label %[[VECTOR_BODY:.*]] ; CHECK-ORDERED-TF: [[VECTOR_BODY]]: ; CHECK-ORDERED-TF-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] -; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ] -; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK4:%.*]] = phi [ [[ACTIVE_LANE_MASK_ENTRY1]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT10:%.*]], %[[VECTOR_BODY]] ] -; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK5:%.*]] = phi [ [[ACTIVE_LANE_MASK_ENTRY2]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT11:%.*]], %[[VECTOR_BODY]] ] -; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK6:%.*]] = phi [ [[ACTIVE_LANE_MASK_ENTRY3]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT12:%.*]], %[[VECTOR_BODY]] ] +; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi [ [[TMP17]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ] +; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK4:%.*]] = phi [ [[TMP18]], %[[VECTOR_PH]] ], [ [[TMP22:%.*]], %[[VECTOR_BODY]] ] +; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK5:%.*]] = phi [ [[TMP19]], %[[VECTOR_PH]] ], [ [[TMP23:%.*]], %[[VECTOR_BODY]] ] +; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK6:%.*]] = phi [ [[TMP25]], %[[VECTOR_PH]] ], [ [[TMP24:%.*]], %[[VECTOR_BODY]] ] ; CHECK-ORDERED-TF-NEXT: [[VEC_PHI:%.*]] = phi float [ 0.000000e+00, %[[VECTOR_PH]] ], [ [[TMP16:%.*]], %[[VECTOR_BODY]] ] ; CHECK-ORDERED-TF-NEXT: [[TMP5:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]] +; CHECK-ORDERED-TF-NEXT: [[TMP3:%.*]] = shl nuw nsw i64 [[TMP1]], 1 +; CHECK-ORDERED-TF-NEXT: [[TMP4:%.*]] = mul nuw nsw i64 [[TMP1]], 3 ; CHECK-ORDERED-TF-NEXT: [[TMP6:%.*]] = getelementptr inbounds float, ptr [[TMP5]], i64 [[TMP1]] ; CHECK-ORDERED-TF-NEXT: [[TMP7:%.*]] = getelementptr inbounds float, ptr [[TMP5]], i64 [[TMP3]] ; CHECK-ORDERED-TF-NEXT: [[TMP8:%.*]] = getelementptr inbounds float, ptr [[TMP5]], i64 [[TMP4]] @@ -331,13 +332,11 @@ define float @fadd_strict_unroll(ptr noalias nocapture readonly %a, i64 %n) #0 { ; CHECK-ORDERED-TF-NEXT: [[TMP15:%.*]] = select [[ACTIVE_LANE_MASK6]], [[WIDE_MASKED_LOAD9]], splat (float -0.000000e+00) ; CHECK-ORDERED-TF-NEXT: [[TMP16]] = call float @llvm.vector.reduce.fadd.nxv8f32(float [[TMP14]], [[TMP15]]) ; CHECK-ORDERED-TF-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP2]] -; CHECK-ORDERED-TF-NEXT: [[TMP17:%.*]] = add i64 [[INDEX_NEXT]], [[TMP1]] -; CHECK-ORDERED-TF-NEXT: [[TMP18:%.*]] = add i64 [[INDEX_NEXT]], [[TMP3]] -; CHECK-ORDERED-TF-NEXT: [[TMP19:%.*]] = add i64 [[INDEX_NEXT]], [[TMP4]] -; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[INDEX_NEXT]], i64 [[N]]) -; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT10]] = call @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP17]], i64 [[N]]) -; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT11]] = call @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP18]], i64 [[N]]) -; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT12]] = call @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP19]], i64 [[N]]) +; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT1:%.*]] = call @llvm.get.active.lane.mask.nxv32i1.i64(i64 [[INDEX_NEXT]], i64 [[N]]) +; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.vector.extract.nxv8i1.nxv32i1( [[ACTIVE_LANE_MASK_NEXT1]], i64 0) +; CHECK-ORDERED-TF-NEXT: [[TMP22]] = call @llvm.vector.extract.nxv8i1.nxv32i1( [[ACTIVE_LANE_MASK_NEXT1]], i64 8) +; CHECK-ORDERED-TF-NEXT: [[TMP23]] = call @llvm.vector.extract.nxv8i1.nxv32i1( [[ACTIVE_LANE_MASK_NEXT1]], i64 16) +; CHECK-ORDERED-TF-NEXT: [[TMP24]] = call @llvm.vector.extract.nxv8i1.nxv32i1( [[ACTIVE_LANE_MASK_NEXT1]], i64 24) ; CHECK-ORDERED-TF-NEXT: [[TMP20:%.*]] = extractelement [[ACTIVE_LANE_MASK_NEXT]], i64 0 ; CHECK-ORDERED-TF-NEXT: [[TMP21:%.*]] = xor i1 [[TMP20]], true ; CHECK-ORDERED-TF-NEXT: br i1 [[TMP21]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]] @@ -1304,21 +1303,22 @@ define float @fmuladd_strict(ptr %a, ptr %b, i64 %n) #0 { ; CHECK-ORDERED-TF-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64() ; CHECK-ORDERED-TF-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3 ; CHECK-ORDERED-TF-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 2 -; CHECK-ORDERED-TF-NEXT: [[TMP3:%.*]] = shl i64 [[TMP1]], 1 -; CHECK-ORDERED-TF-NEXT: [[TMP4:%.*]] = mul i64 [[TMP1]], 3 -; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 [[N]]) -; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_ENTRY1:%.*]] = call @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP1]], i64 [[N]]) -; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_ENTRY2:%.*]] = call @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP3]], i64 [[N]]) -; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_ENTRY3:%.*]] = call @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP4]], i64 [[N]]) +; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call @llvm.get.active.lane.mask.nxv32i1.i64(i64 0, i64 [[N]]) +; CHECK-ORDERED-TF-NEXT: [[TMP25:%.*]] = call @llvm.vector.extract.nxv8i1.nxv32i1( [[ACTIVE_LANE_MASK_ENTRY]], i64 0) +; CHECK-ORDERED-TF-NEXT: [[TMP26:%.*]] = call @llvm.vector.extract.nxv8i1.nxv32i1( [[ACTIVE_LANE_MASK_ENTRY]], i64 8) +; CHECK-ORDERED-TF-NEXT: [[TMP27:%.*]] = call @llvm.vector.extract.nxv8i1.nxv32i1( [[ACTIVE_LANE_MASK_ENTRY]], i64 16) +; CHECK-ORDERED-TF-NEXT: [[TMP33:%.*]] = call @llvm.vector.extract.nxv8i1.nxv32i1( [[ACTIVE_LANE_MASK_ENTRY]], i64 24) ; CHECK-ORDERED-TF-NEXT: br label %[[VECTOR_BODY:.*]] ; CHECK-ORDERED-TF: [[VECTOR_BODY]]: ; CHECK-ORDERED-TF-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] -; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ] -; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK4:%.*]] = phi [ [[ACTIVE_LANE_MASK_ENTRY1]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT14:%.*]], %[[VECTOR_BODY]] ] -; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK5:%.*]] = phi [ [[ACTIVE_LANE_MASK_ENTRY2]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT15:%.*]], %[[VECTOR_BODY]] ] -; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK6:%.*]] = phi [ [[ACTIVE_LANE_MASK_ENTRY3]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT16:%.*]], %[[VECTOR_BODY]] ] +; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi [ [[TMP25]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ] +; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK4:%.*]] = phi [ [[TMP26]], %[[VECTOR_PH]] ], [ [[TMP30:%.*]], %[[VECTOR_BODY]] ] +; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK5:%.*]] = phi [ [[TMP27]], %[[VECTOR_PH]] ], [ [[TMP31:%.*]], %[[VECTOR_BODY]] ] +; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK6:%.*]] = phi [ [[TMP33]], %[[VECTOR_PH]] ], [ [[TMP32:%.*]], %[[VECTOR_BODY]] ] ; CHECK-ORDERED-TF-NEXT: [[VEC_PHI:%.*]] = phi float [ 0.000000e+00, %[[VECTOR_PH]] ], [ [[TMP24:%.*]], %[[VECTOR_BODY]] ] ; CHECK-ORDERED-TF-NEXT: [[TMP5:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]] +; CHECK-ORDERED-TF-NEXT: [[TMP3:%.*]] = shl nuw nsw i64 [[TMP1]], 1 +; CHECK-ORDERED-TF-NEXT: [[TMP4:%.*]] = mul nuw nsw i64 [[TMP1]], 3 ; CHECK-ORDERED-TF-NEXT: [[TMP6:%.*]] = getelementptr inbounds float, ptr [[TMP5]], i64 [[TMP1]] ; CHECK-ORDERED-TF-NEXT: [[TMP7:%.*]] = getelementptr inbounds float, ptr [[TMP5]], i64 [[TMP3]] ; CHECK-ORDERED-TF-NEXT: [[TMP8:%.*]] = getelementptr inbounds float, ptr [[TMP5]], i64 [[TMP4]] @@ -1347,13 +1347,11 @@ define float @fmuladd_strict(ptr %a, ptr %b, i64 %n) #0 { ; CHECK-ORDERED-TF-NEXT: [[TMP23:%.*]] = select [[ACTIVE_LANE_MASK6]], [[TMP16]], splat (float -0.000000e+00) ; CHECK-ORDERED-TF-NEXT: [[TMP24]] = call float @llvm.vector.reduce.fadd.nxv8f32(float [[TMP22]], [[TMP23]]) ; CHECK-ORDERED-TF-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP2]] -; CHECK-ORDERED-TF-NEXT: [[TMP25:%.*]] = add i64 [[INDEX_NEXT]], [[TMP1]] -; CHECK-ORDERED-TF-NEXT: [[TMP26:%.*]] = add i64 [[INDEX_NEXT]], [[TMP3]] -; CHECK-ORDERED-TF-NEXT: [[TMP27:%.*]] = add i64 [[INDEX_NEXT]], [[TMP4]] -; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[INDEX_NEXT]], i64 [[N]]) -; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT14]] = call @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP25]], i64 [[N]]) -; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT15]] = call @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP26]], i64 [[N]]) -; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT16]] = call @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP27]], i64 [[N]]) +; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT1:%.*]] = call @llvm.get.active.lane.mask.nxv32i1.i64(i64 [[INDEX_NEXT]], i64 [[N]]) +; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.vector.extract.nxv8i1.nxv32i1( [[ACTIVE_LANE_MASK_NEXT1]], i64 0) +; CHECK-ORDERED-TF-NEXT: [[TMP30]] = call @llvm.vector.extract.nxv8i1.nxv32i1( [[ACTIVE_LANE_MASK_NEXT1]], i64 8) +; CHECK-ORDERED-TF-NEXT: [[TMP31]] = call @llvm.vector.extract.nxv8i1.nxv32i1( [[ACTIVE_LANE_MASK_NEXT1]], i64 16) +; CHECK-ORDERED-TF-NEXT: [[TMP32]] = call @llvm.vector.extract.nxv8i1.nxv32i1( [[ACTIVE_LANE_MASK_NEXT1]], i64 24) ; CHECK-ORDERED-TF-NEXT: [[TMP28:%.*]] = extractelement [[ACTIVE_LANE_MASK_NEXT]], i64 0 ; CHECK-ORDERED-TF-NEXT: [[TMP29:%.*]] = xor i1 [[TMP28]], true ; CHECK-ORDERED-TF-NEXT: br i1 [[TMP29]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]] @@ -1551,21 +1549,22 @@ define float @fmuladd_strict_fmf(ptr %a, ptr %b, i64 %n) #0 { ; CHECK-ORDERED-TF-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64() ; CHECK-ORDERED-TF-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3 ; CHECK-ORDERED-TF-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 2 -; CHECK-ORDERED-TF-NEXT: [[TMP3:%.*]] = shl i64 [[TMP1]], 1 -; CHECK-ORDERED-TF-NEXT: [[TMP4:%.*]] = mul i64 [[TMP1]], 3 -; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 [[N]]) -; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_ENTRY1:%.*]] = call @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP1]], i64 [[N]]) -; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_ENTRY2:%.*]] = call @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP3]], i64 [[N]]) -; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_ENTRY3:%.*]] = call @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP4]], i64 [[N]]) +; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call @llvm.get.active.lane.mask.nxv32i1.i64(i64 0, i64 [[N]]) +; CHECK-ORDERED-TF-NEXT: [[TMP25:%.*]] = call @llvm.vector.extract.nxv8i1.nxv32i1( [[ACTIVE_LANE_MASK_ENTRY]], i64 0) +; CHECK-ORDERED-TF-NEXT: [[TMP26:%.*]] = call @llvm.vector.extract.nxv8i1.nxv32i1( [[ACTIVE_LANE_MASK_ENTRY]], i64 8) +; CHECK-ORDERED-TF-NEXT: [[TMP27:%.*]] = call @llvm.vector.extract.nxv8i1.nxv32i1( [[ACTIVE_LANE_MASK_ENTRY]], i64 16) +; CHECK-ORDERED-TF-NEXT: [[TMP33:%.*]] = call @llvm.vector.extract.nxv8i1.nxv32i1( [[ACTIVE_LANE_MASK_ENTRY]], i64 24) ; CHECK-ORDERED-TF-NEXT: br label %[[VECTOR_BODY:.*]] ; CHECK-ORDERED-TF: [[VECTOR_BODY]]: ; CHECK-ORDERED-TF-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] -; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ] -; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK4:%.*]] = phi [ [[ACTIVE_LANE_MASK_ENTRY1]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT14:%.*]], %[[VECTOR_BODY]] ] -; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK5:%.*]] = phi [ [[ACTIVE_LANE_MASK_ENTRY2]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT15:%.*]], %[[VECTOR_BODY]] ] -; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK6:%.*]] = phi [ [[ACTIVE_LANE_MASK_ENTRY3]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT16:%.*]], %[[VECTOR_BODY]] ] +; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi [ [[TMP25]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ] +; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK4:%.*]] = phi [ [[TMP26]], %[[VECTOR_PH]] ], [ [[TMP30:%.*]], %[[VECTOR_BODY]] ] +; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK5:%.*]] = phi [ [[TMP27]], %[[VECTOR_PH]] ], [ [[TMP31:%.*]], %[[VECTOR_BODY]] ] +; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK6:%.*]] = phi [ [[TMP33]], %[[VECTOR_PH]] ], [ [[TMP32:%.*]], %[[VECTOR_BODY]] ] ; CHECK-ORDERED-TF-NEXT: [[VEC_PHI:%.*]] = phi float [ 0.000000e+00, %[[VECTOR_PH]] ], [ [[TMP24:%.*]], %[[VECTOR_BODY]] ] ; CHECK-ORDERED-TF-NEXT: [[TMP5:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]] +; CHECK-ORDERED-TF-NEXT: [[TMP3:%.*]] = shl nuw nsw i64 [[TMP1]], 1 +; CHECK-ORDERED-TF-NEXT: [[TMP4:%.*]] = mul nuw nsw i64 [[TMP1]], 3 ; CHECK-ORDERED-TF-NEXT: [[TMP6:%.*]] = getelementptr inbounds float, ptr [[TMP5]], i64 [[TMP1]] ; CHECK-ORDERED-TF-NEXT: [[TMP7:%.*]] = getelementptr inbounds float, ptr [[TMP5]], i64 [[TMP3]] ; CHECK-ORDERED-TF-NEXT: [[TMP8:%.*]] = getelementptr inbounds float, ptr [[TMP5]], i64 [[TMP4]] @@ -1594,13 +1593,11 @@ define float @fmuladd_strict_fmf(ptr %a, ptr %b, i64 %n) #0 { ; CHECK-ORDERED-TF-NEXT: [[TMP23:%.*]] = select nnan [[ACTIVE_LANE_MASK6]], [[TMP16]], splat (float -0.000000e+00) ; CHECK-ORDERED-TF-NEXT: [[TMP24]] = call nnan float @llvm.vector.reduce.fadd.nxv8f32(float [[TMP22]], [[TMP23]]) ; CHECK-ORDERED-TF-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP2]] -; CHECK-ORDERED-TF-NEXT: [[TMP25:%.*]] = add i64 [[INDEX_NEXT]], [[TMP1]] -; CHECK-ORDERED-TF-NEXT: [[TMP26:%.*]] = add i64 [[INDEX_NEXT]], [[TMP3]] -; CHECK-ORDERED-TF-NEXT: [[TMP27:%.*]] = add i64 [[INDEX_NEXT]], [[TMP4]] -; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[INDEX_NEXT]], i64 [[N]]) -; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT14]] = call @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP25]], i64 [[N]]) -; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT15]] = call @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP26]], i64 [[N]]) -; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT16]] = call @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP27]], i64 [[N]]) +; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT1:%.*]] = call @llvm.get.active.lane.mask.nxv32i1.i64(i64 [[INDEX_NEXT]], i64 [[N]]) +; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.vector.extract.nxv8i1.nxv32i1( [[ACTIVE_LANE_MASK_NEXT1]], i64 0) +; CHECK-ORDERED-TF-NEXT: [[TMP30]] = call @llvm.vector.extract.nxv8i1.nxv32i1( [[ACTIVE_LANE_MASK_NEXT1]], i64 8) +; CHECK-ORDERED-TF-NEXT: [[TMP31]] = call @llvm.vector.extract.nxv8i1.nxv32i1( [[ACTIVE_LANE_MASK_NEXT1]], i64 16) +; CHECK-ORDERED-TF-NEXT: [[TMP32]] = call @llvm.vector.extract.nxv8i1.nxv32i1( [[ACTIVE_LANE_MASK_NEXT1]], i64 24) ; CHECK-ORDERED-TF-NEXT: [[TMP28:%.*]] = extractelement [[ACTIVE_LANE_MASK_NEXT]], i64 0 ; CHECK-ORDERED-TF-NEXT: [[TMP29:%.*]] = xor i1 [[TMP28]], true ; CHECK-ORDERED-TF-NEXT: br i1 [[TMP29]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]] diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-unroll.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-unroll.ll index 83d9a717d04d2..55b7b14c6139e 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-unroll.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-unroll.ll @@ -13,12 +13,11 @@ define void @simple_memset(i32 %val, ptr %ptr, i64 %n) #0 { ; CHECK-NEXT: [[TMP61:%.*]] = call i64 @llvm.vscale.i64() ; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP61]], 2 ; CHECK-NEXT: [[TMP62:%.*]] = shl nuw i64 [[TMP1]], 2 -; CHECK-NEXT: [[TMP28:%.*]] = shl i64 [[TMP1]], 1 -; CHECK-NEXT: [[TMP30:%.*]] = mul i64 [[TMP1]], 3 -; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[UMAX]]) -; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY3:%.*]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[TMP1]], i64 [[UMAX]]) -; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY4:%.*]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[TMP28]], i64 [[UMAX]]) -; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY5:%.*]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[TMP30]], i64 [[UMAX]]) +; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY1:%.*]] = call @llvm.get.active.lane.mask.nxv16i1.i64(i64 0, i64 [[UMAX]]) +; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call @llvm.vector.extract.nxv4i1.nxv16i1( [[ACTIVE_LANE_MASK_ENTRY1]], i64 0) +; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY3:%.*]] = call @llvm.vector.extract.nxv4i1.nxv16i1( [[ACTIVE_LANE_MASK_ENTRY1]], i64 4) +; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY4:%.*]] = call @llvm.vector.extract.nxv4i1.nxv16i1( [[ACTIVE_LANE_MASK_ENTRY1]], i64 8) +; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY5:%.*]] = call @llvm.vector.extract.nxv4i1.nxv16i1( [[ACTIVE_LANE_MASK_ENTRY1]], i64 12) ; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement poison, i32 [[VAL:%.*]], i64 0 ; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector [[BROADCAST_SPLATINSERT]], poison, zeroinitializer ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] @@ -29,6 +28,8 @@ define void @simple_memset(i32 %val, ptr %ptr, i64 %n) #0 { ; CHECK-NEXT: [[ACTIVE_LANE_MASK8:%.*]] = phi [ [[ACTIVE_LANE_MASK_ENTRY4]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT12:%.*]], [[VECTOR_BODY]] ] ; CHECK-NEXT: [[ACTIVE_LANE_MASK9:%.*]] = phi [ [[ACTIVE_LANE_MASK_ENTRY5]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT13:%.*]], [[VECTOR_BODY]] ] ; CHECK-NEXT: [[TMP47:%.*]] = getelementptr i32, ptr [[PTR:%.*]], i64 [[INDEX6]] +; CHECK-NEXT: [[TMP28:%.*]] = shl nuw nsw i64 [[TMP1]], 1 +; CHECK-NEXT: [[TMP30:%.*]] = mul nuw nsw i64 [[TMP1]], 3 ; CHECK-NEXT: [[TMP54:%.*]] = getelementptr i32, ptr [[TMP47]], i64 [[TMP1]] ; CHECK-NEXT: [[TMP57:%.*]] = getelementptr i32, ptr [[TMP47]], i64 [[TMP28]] ; CHECK-NEXT: [[TMP60:%.*]] = getelementptr i32, ptr [[TMP47]], i64 [[TMP30]] @@ -37,13 +38,11 @@ define void @simple_memset(i32 %val, ptr %ptr, i64 %n) #0 { ; CHECK-NEXT: call void @llvm.masked.store.nxv4i32.p0( [[BROADCAST_SPLAT]], ptr align 4 [[TMP57]], [[ACTIVE_LANE_MASK8]]) ; CHECK-NEXT: call void @llvm.masked.store.nxv4i32.p0( [[BROADCAST_SPLAT]], ptr align 4 [[TMP60]], [[ACTIVE_LANE_MASK9]]) ; CHECK-NEXT: [[INDEX_NEXT10]] = add i64 [[INDEX6]], [[TMP62]] -; CHECK-NEXT: [[TMP9:%.*]] = add i64 [[INDEX_NEXT10]], [[TMP1]] -; CHECK-NEXT: [[TMP10:%.*]] = add i64 [[INDEX_NEXT10]], [[TMP28]] -; CHECK-NEXT: [[TMP11:%.*]] = add i64 [[INDEX_NEXT10]], [[TMP30]] -; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT10]], i64 [[UMAX]]) -; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT11]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[TMP9]], i64 [[UMAX]]) -; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT12]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[TMP10]], i64 [[UMAX]]) -; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT13]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[TMP11]], i64 [[UMAX]]) +; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT1:%.*]] = call @llvm.get.active.lane.mask.nxv16i1.i64(i64 [[INDEX_NEXT10]], i64 [[UMAX]]) +; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.vector.extract.nxv4i1.nxv16i1( [[ACTIVE_LANE_MASK_NEXT1]], i64 0) +; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT11]] = call @llvm.vector.extract.nxv4i1.nxv16i1( [[ACTIVE_LANE_MASK_NEXT1]], i64 4) +; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT12]] = call @llvm.vector.extract.nxv4i1.nxv16i1( [[ACTIVE_LANE_MASK_NEXT1]], i64 8) +; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT13]] = call @llvm.vector.extract.nxv4i1.nxv16i1( [[ACTIVE_LANE_MASK_NEXT1]], i64 12) ; CHECK-NEXT: [[TMP35:%.*]] = extractelement [[ACTIVE_LANE_MASK_NEXT]], i64 0 ; CHECK-NEXT: [[TMP36:%.*]] = xor i1 [[TMP35]], true ; CHECK-NEXT: br i1 [[TMP36]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] @@ -76,12 +75,11 @@ define void @cond_memset(i32 %val, ptr noalias readonly %cond_ptr, ptr noalias % ; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64() ; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP5]], 2 ; CHECK-NEXT: [[TMP6:%.*]] = shl nuw i64 [[TMP1]], 2 -; CHECK-NEXT: [[TMP28:%.*]] = shl i64 [[TMP1]], 1 -; CHECK-NEXT: [[TMP30:%.*]] = mul i64 [[TMP1]], 3 -; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[UMAX]]) -; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY3:%.*]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[TMP1]], i64 [[UMAX]]) -; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY4:%.*]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[TMP28]], i64 [[UMAX]]) -; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY5:%.*]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[TMP30]], i64 [[UMAX]]) +; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY1:%.*]] = call @llvm.get.active.lane.mask.nxv16i1.i64(i64 0, i64 [[UMAX]]) +; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call @llvm.vector.extract.nxv4i1.nxv16i1( [[ACTIVE_LANE_MASK_ENTRY1]], i64 0) +; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY3:%.*]] = call @llvm.vector.extract.nxv4i1.nxv16i1( [[ACTIVE_LANE_MASK_ENTRY1]], i64 4) +; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY4:%.*]] = call @llvm.vector.extract.nxv4i1.nxv16i1( [[ACTIVE_LANE_MASK_ENTRY1]], i64 8) +; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY5:%.*]] = call @llvm.vector.extract.nxv4i1.nxv16i1( [[ACTIVE_LANE_MASK_ENTRY1]], i64 12) ; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement poison, i32 [[VAL:%.*]], i64 0 ; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector [[BROADCAST_SPLATINSERT]], poison, zeroinitializer ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] @@ -92,6 +90,8 @@ define void @cond_memset(i32 %val, ptr noalias readonly %cond_ptr, ptr noalias % ; CHECK-NEXT: [[ACTIVE_LANE_MASK8:%.*]] = phi [ [[ACTIVE_LANE_MASK_ENTRY4]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT15:%.*]], [[VECTOR_BODY]] ] ; CHECK-NEXT: [[ACTIVE_LANE_MASK9:%.*]] = phi [ [[ACTIVE_LANE_MASK_ENTRY5]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT16:%.*]], [[VECTOR_BODY]] ] ; CHECK-NEXT: [[TMP47:%.*]] = getelementptr i32, ptr [[COND_PTR:%.*]], i64 [[INDEX6]] +; CHECK-NEXT: [[TMP28:%.*]] = shl nuw nsw i64 [[TMP1]], 1 +; CHECK-NEXT: [[TMP30:%.*]] = mul nuw nsw i64 [[TMP1]], 3 ; CHECK-NEXT: [[TMP54:%.*]] = getelementptr i32, ptr [[TMP47]], i64 [[TMP1]] ; CHECK-NEXT: [[TMP57:%.*]] = getelementptr i32, ptr [[TMP47]], i64 [[TMP28]] ; CHECK-NEXT: [[TMP60:%.*]] = getelementptr i32, ptr [[TMP47]], i64 [[TMP30]] @@ -116,13 +116,11 @@ define void @cond_memset(i32 %val, ptr noalias readonly %cond_ptr, ptr noalias % ; CHECK-NEXT: call void @llvm.masked.store.nxv4i32.p0( [[BROADCAST_SPLAT]], ptr align 4 [[TMP79]], [[TMP71]]) ; CHECK-NEXT: call void @llvm.masked.store.nxv4i32.p0( [[BROADCAST_SPLAT]], ptr align 4 [[TMP82]], [[TMP72]]) ; CHECK-NEXT: [[INDEX_NEXT13]] = add i64 [[INDEX6]], [[TMP6]] -; CHECK-NEXT: [[TMP21:%.*]] = add i64 [[INDEX_NEXT13]], [[TMP1]] -; CHECK-NEXT: [[TMP22:%.*]] = add i64 [[INDEX_NEXT13]], [[TMP28]] -; CHECK-NEXT: [[TMP23:%.*]] = add i64 [[INDEX_NEXT13]], [[TMP30]] -; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT13]], i64 [[UMAX]]) -; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT14]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[TMP21]], i64 [[UMAX]]) -; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT15]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[TMP22]], i64 [[UMAX]]) -; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT16]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[TMP23]], i64 [[UMAX]]) +; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT1:%.*]] = call @llvm.get.active.lane.mask.nxv16i1.i64(i64 [[INDEX_NEXT13]], i64 [[UMAX]]) +; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.vector.extract.nxv4i1.nxv16i1( [[ACTIVE_LANE_MASK_NEXT1]], i64 0) +; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT14]] = call @llvm.vector.extract.nxv4i1.nxv16i1( [[ACTIVE_LANE_MASK_NEXT1]], i64 4) +; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT15]] = call @llvm.vector.extract.nxv4i1.nxv16i1( [[ACTIVE_LANE_MASK_NEXT1]], i64 8) +; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT16]] = call @llvm.vector.extract.nxv4i1.nxv16i1( [[ACTIVE_LANE_MASK_NEXT1]], i64 12) ; CHECK-NEXT: [[TMP66:%.*]] = extractelement [[ACTIVE_LANE_MASK_NEXT]], i64 0 ; CHECK-NEXT: [[TMP67:%.*]] = xor i1 [[TMP66]], true ; CHECK-NEXT: br i1 [[TMP67]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]] diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-wide-lane-mask.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-wide-lane-mask.ll index 2e6e3a41addc5..8c213d3aa528f 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-wide-lane-mask.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-wide-lane-mask.ll @@ -1,8 +1,7 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "^middle.block:" --version 4 -; RUN: opt -S --passes=loop-vectorize -tail-folding-policy=must-fold-tail -enable-wide-lane-mask -force-vector-interleave=1 < %s | FileCheck %s -check-prefix CHECK-UF1 -; RUN: opt -S --passes=loop-vectorize -tail-folding-policy=must-fold-tail -enable-wide-lane-mask -force-vector-interleave=4 < %s | FileCheck %s -check-prefix CHECK-UF4 -; RUN: opt -S --passes=loop-vectorize -enable-wide-lane-mask -tail-folding-policy=must-fold-tail < %s | FileCheck %s -check-prefix CHECK-TF -; RUN: opt -S --passes=forceattrs,loop-vectorize -enable-wide-lane-mask -tail-folding-policy=must-fold-tail -force-attribute=optsize < %s | FileCheck %s -check-prefix CHECK-UF1 +; RUN: opt -S --passes=loop-vectorize -tail-folding-policy=must-fold-tail -force-vector-interleave=1 < %s | FileCheck %s -check-prefix CHECK-UF1 +; RUN: opt -S --passes=loop-vectorize -tail-folding-policy=must-fold-tail -force-vector-interleave=4 < %s | FileCheck %s -check-prefix CHECK-UF4 +; RUN: opt -S --passes=forceattrs,loop-vectorize -tail-folding-policy=must-fold-tail -force-attribute=optsize < %s | FileCheck %s -check-prefix CHECK-UF1 target triple = "aarch64-unknown-linux" @@ -10,115 +9,80 @@ define void @scalable_wide_active_lane_mask(ptr noalias %dst, ptr readonly %src, ; CHECK-UF1-LABEL: define void @scalable_wide_active_lane_mask( ; CHECK-UF1-SAME: ptr noalias [[DST:%.*]], ptr readonly [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] { ; CHECK-UF1-NEXT: entry: -; CHECK-UF1-NEXT: br label [[VECTOR_PH1:%.*]] +; CHECK-UF1-NEXT: br label [[VECTOR_PH:%.*]] ; CHECK-UF1: vector.ph: -; CHECK-UF1-NEXT: [[TMP17:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-UF1-NEXT: [[TMP18:%.*]] = shl nuw i64 [[TMP17]], 4 +; CHECK-UF1-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-UF1-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 4 ; CHECK-UF1-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call @llvm.get.active.lane.mask.nxv16i1.i64(i64 0, i64 [[N]]) ; CHECK-UF1-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK-UF1: vector.body: -; CHECK-UF1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH1]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-UF1-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH1]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-UF1-NEXT: [[TMP10:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 [[INDEX]] -; CHECK-UF1-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP10]], [[ACTIVE_LANE_MASK]], poison) -; CHECK-UF1-NEXT: [[TMP6:%.*]] = mul [[WIDE_MASKED_LOAD]], splat (i8 3) -; CHECK-UF1-NEXT: [[TMP13:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[INDEX]] -; CHECK-UF1-NEXT: call void @llvm.masked.store.nxv16i8.p0( [[TMP6]], ptr align 1 [[TMP13]], [[ACTIVE_LANE_MASK]]) -; CHECK-UF1-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP18]] +; CHECK-UF1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] +; CHECK-UF1-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ] +; CHECK-UF1-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 [[INDEX]] +; CHECK-UF1-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP3]], [[ACTIVE_LANE_MASK]], poison) +; CHECK-UF1-NEXT: [[TMP4:%.*]] = mul [[WIDE_MASKED_LOAD]], splat (i8 3) +; CHECK-UF1-NEXT: [[TMP5:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[INDEX]] +; CHECK-UF1-NEXT: call void @llvm.masked.store.nxv16i8.p0( [[TMP4]], ptr align 1 [[TMP5]], [[ACTIVE_LANE_MASK]]) +; CHECK-UF1-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP1]] ; CHECK-UF1-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.get.active.lane.mask.nxv16i1.i64(i64 [[INDEX_NEXT]], i64 [[N]]) -; CHECK-UF1-NEXT: [[TMP14:%.*]] = extractelement [[ACTIVE_LANE_MASK_NEXT]], i64 0 -; CHECK-UF1-NEXT: [[TMP11:%.*]] = xor i1 [[TMP14]], true -; CHECK-UF1-NEXT: br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] +; CHECK-UF1-NEXT: [[TMP6:%.*]] = extractelement [[ACTIVE_LANE_MASK_NEXT]], i64 0 +; CHECK-UF1-NEXT: [[TMP7:%.*]] = xor i1 [[TMP6]], true +; CHECK-UF1-NEXT: br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] ; CHECK-UF1: middle.block: ; ; CHECK-UF4-LABEL: define void @scalable_wide_active_lane_mask( ; CHECK-UF4-SAME: ptr noalias [[DST:%.*]], ptr readonly [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] { ; CHECK-UF4-NEXT: entry: -; CHECK-UF4-NEXT: br label [[VECTOR_PH1:%.*]] +; CHECK-UF4-NEXT: br label [[VECTOR_PH:%.*]] ; CHECK-UF4: vector.ph: -; CHECK-UF4-NEXT: [[TMP61:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-UF4-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP61]], 4 -; CHECK-UF4-NEXT: [[TMP62:%.*]] = shl nuw i64 [[TMP1]], 2 +; CHECK-UF4-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-UF4-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 4 +; CHECK-UF4-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 2 ; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call @llvm.get.active.lane.mask.nxv64i1.i64(i64 0, i64 [[N]]) -; CHECK-UF4-NEXT: [[TMP19:%.*]] = call @llvm.vector.extract.nxv16i1.nxv64i1( [[ACTIVE_LANE_MASK_ENTRY]], i64 48) -; CHECK-UF4-NEXT: [[TMP18:%.*]] = call @llvm.vector.extract.nxv16i1.nxv64i1( [[ACTIVE_LANE_MASK_ENTRY]], i64 32) -; CHECK-UF4-NEXT: [[TMP17:%.*]] = call @llvm.vector.extract.nxv16i1.nxv64i1( [[ACTIVE_LANE_MASK_ENTRY]], i64 16) -; CHECK-UF4-NEXT: [[TMP16:%.*]] = call @llvm.vector.extract.nxv16i1.nxv64i1( [[ACTIVE_LANE_MASK_ENTRY]], i64 0) +; CHECK-UF4-NEXT: [[TMP3:%.*]] = call @llvm.vector.extract.nxv16i1.nxv64i1( [[ACTIVE_LANE_MASK_ENTRY]], i64 0) +; CHECK-UF4-NEXT: [[EXTRACT_ENTRY_ALM_PART1:%.*]] = call @llvm.vector.extract.nxv16i1.nxv64i1( [[ACTIVE_LANE_MASK_ENTRY]], i64 16) +; CHECK-UF4-NEXT: [[EXTRACT_ENTRY_ALM_PART2:%.*]] = call @llvm.vector.extract.nxv16i1.nxv64i1( [[ACTIVE_LANE_MASK_ENTRY]], i64 32) +; CHECK-UF4-NEXT: [[EXTRACT_ENTRY_ALM_PART3:%.*]] = call @llvm.vector.extract.nxv16i1.nxv64i1( [[ACTIVE_LANE_MASK_ENTRY]], i64 48) ; CHECK-UF4-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK-UF4: vector.body: -; CHECK-UF4-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH1]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi [ [[TMP16]], [[VECTOR_PH1]] ], [ [[TMP55:%.*]], [[VECTOR_BODY]] ] -; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK6:%.*]] = phi [ [[TMP17]], [[VECTOR_PH1]] ], [ [[TMP56:%.*]], [[VECTOR_BODY]] ] -; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK7:%.*]] = phi [ [[TMP18]], [[VECTOR_PH1]] ], [ [[TMP57:%.*]], [[VECTOR_BODY]] ] -; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK8:%.*]] = phi [ [[TMP19]], [[VECTOR_PH1]] ], [ [[TMP58:%.*]], [[VECTOR_BODY]] ] -; CHECK-UF4-NEXT: [[TMP20:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 [[INDEX]] -; CHECK-UF4-NEXT: [[TMP32:%.*]] = shl nuw nsw i64 [[TMP1]], 1 -; CHECK-UF4-NEXT: [[TMP29:%.*]] = mul nuw nsw i64 [[TMP1]], 3 -; CHECK-UF4-NEXT: [[TMP24:%.*]] = getelementptr inbounds i8, ptr [[TMP20]], i64 [[TMP1]] -; CHECK-UF4-NEXT: [[TMP33:%.*]] = getelementptr inbounds i8, ptr [[TMP20]], i64 [[TMP32]] -; CHECK-UF4-NEXT: [[TMP30:%.*]] = getelementptr inbounds i8, ptr [[TMP20]], i64 [[TMP29]] -; CHECK-UF4-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP20]], [[ACTIVE_LANE_MASK]], poison) -; CHECK-UF4-NEXT: [[WIDE_MASKED_LOAD9:%.*]] = call @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP24]], [[ACTIVE_LANE_MASK6]], poison) -; CHECK-UF4-NEXT: [[WIDE_MASKED_LOAD10:%.*]] = call @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP33]], [[ACTIVE_LANE_MASK7]], poison) -; CHECK-UF4-NEXT: [[WIDE_MASKED_LOAD11:%.*]] = call @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP30]], [[ACTIVE_LANE_MASK8]], poison) -; CHECK-UF4-NEXT: [[TMP25:%.*]] = mul [[WIDE_MASKED_LOAD]], splat (i8 3) -; CHECK-UF4-NEXT: [[TMP26:%.*]] = mul [[WIDE_MASKED_LOAD9]], splat (i8 3) -; CHECK-UF4-NEXT: [[TMP27:%.*]] = mul [[WIDE_MASKED_LOAD10]], splat (i8 3) -; CHECK-UF4-NEXT: [[TMP28:%.*]] = mul [[WIDE_MASKED_LOAD11]], splat (i8 3) -; CHECK-UF4-NEXT: [[TMP35:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[INDEX]] -; CHECK-UF4-NEXT: [[TMP39:%.*]] = getelementptr inbounds i8, ptr [[TMP35]], i64 [[TMP1]] -; CHECK-UF4-NEXT: [[TMP42:%.*]] = getelementptr inbounds i8, ptr [[TMP35]], i64 [[TMP32]] -; CHECK-UF4-NEXT: [[TMP45:%.*]] = getelementptr inbounds i8, ptr [[TMP35]], i64 [[TMP29]] -; CHECK-UF4-NEXT: call void @llvm.masked.store.nxv16i8.p0( [[TMP25]], ptr align 1 [[TMP35]], [[ACTIVE_LANE_MASK]]) -; CHECK-UF4-NEXT: call void @llvm.masked.store.nxv16i8.p0( [[TMP26]], ptr align 1 [[TMP39]], [[ACTIVE_LANE_MASK6]]) -; CHECK-UF4-NEXT: call void @llvm.masked.store.nxv16i8.p0( [[TMP27]], ptr align 1 [[TMP42]], [[ACTIVE_LANE_MASK7]]) -; CHECK-UF4-NEXT: call void @llvm.masked.store.nxv16i8.p0( [[TMP28]], ptr align 1 [[TMP45]], [[ACTIVE_LANE_MASK8]]) -; CHECK-UF4-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP62]] +; CHECK-UF4-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] +; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi [ [[TMP3]], [[VECTOR_PH]] ], [ [[TMP21:%.*]], [[VECTOR_BODY]] ] +; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK1:%.*]] = phi [ [[EXTRACT_ENTRY_ALM_PART1]], [[VECTOR_PH]] ], [ [[TMP22:%.*]], [[VECTOR_BODY]] ] +; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK2:%.*]] = phi [ [[EXTRACT_ENTRY_ALM_PART2]], [[VECTOR_PH]] ], [ [[TMP23:%.*]], [[VECTOR_BODY]] ] +; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK3:%.*]] = phi [ [[EXTRACT_ENTRY_ALM_PART3]], [[VECTOR_PH]] ], [ [[TMP24:%.*]], [[VECTOR_BODY]] ] +; CHECK-UF4-NEXT: [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 [[INDEX]] +; CHECK-UF4-NEXT: [[TMP8:%.*]] = shl nuw nsw i64 [[TMP1]], 1 +; CHECK-UF4-NEXT: [[TMP9:%.*]] = mul nuw nsw i64 [[TMP1]], 3 +; CHECK-UF4-NEXT: [[TMP10:%.*]] = getelementptr inbounds i8, ptr [[TMP7]], i64 [[TMP1]] +; CHECK-UF4-NEXT: [[TMP11:%.*]] = getelementptr inbounds i8, ptr [[TMP7]], i64 [[TMP8]] +; CHECK-UF4-NEXT: [[TMP12:%.*]] = getelementptr inbounds i8, ptr [[TMP7]], i64 [[TMP9]] +; CHECK-UF4-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP7]], [[ACTIVE_LANE_MASK]], poison) +; CHECK-UF4-NEXT: [[WIDE_MASKED_LOAD4:%.*]] = call @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP10]], [[ACTIVE_LANE_MASK1]], poison) +; CHECK-UF4-NEXT: [[WIDE_MASKED_LOAD5:%.*]] = call @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP11]], [[ACTIVE_LANE_MASK2]], poison) +; CHECK-UF4-NEXT: [[WIDE_MASKED_LOAD6:%.*]] = call @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP12]], [[ACTIVE_LANE_MASK3]], poison) +; CHECK-UF4-NEXT: [[TMP13:%.*]] = mul [[WIDE_MASKED_LOAD]], splat (i8 3) +; CHECK-UF4-NEXT: [[TMP14:%.*]] = mul [[WIDE_MASKED_LOAD4]], splat (i8 3) +; CHECK-UF4-NEXT: [[TMP15:%.*]] = mul [[WIDE_MASKED_LOAD5]], splat (i8 3) +; CHECK-UF4-NEXT: [[TMP16:%.*]] = mul [[WIDE_MASKED_LOAD6]], splat (i8 3) +; CHECK-UF4-NEXT: [[TMP17:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[INDEX]] +; CHECK-UF4-NEXT: [[TMP18:%.*]] = getelementptr inbounds i8, ptr [[TMP17]], i64 [[TMP1]] +; CHECK-UF4-NEXT: [[TMP19:%.*]] = getelementptr inbounds i8, ptr [[TMP17]], i64 [[TMP8]] +; CHECK-UF4-NEXT: [[TMP20:%.*]] = getelementptr inbounds i8, ptr [[TMP17]], i64 [[TMP9]] +; CHECK-UF4-NEXT: call void @llvm.masked.store.nxv16i8.p0( [[TMP13]], ptr align 1 [[TMP17]], [[ACTIVE_LANE_MASK]]) +; CHECK-UF4-NEXT: call void @llvm.masked.store.nxv16i8.p0( [[TMP14]], ptr align 1 [[TMP18]], [[ACTIVE_LANE_MASK1]]) +; CHECK-UF4-NEXT: call void @llvm.masked.store.nxv16i8.p0( [[TMP15]], ptr align 1 [[TMP19]], [[ACTIVE_LANE_MASK2]]) +; CHECK-UF4-NEXT: call void @llvm.masked.store.nxv16i8.p0( [[TMP16]], ptr align 1 [[TMP20]], [[ACTIVE_LANE_MASK3]]) +; CHECK-UF4-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP2]] ; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK_NEXT:%.*]] = call @llvm.get.active.lane.mask.nxv64i1.i64(i64 [[INDEX_NEXT]], i64 [[N]]) -; CHECK-UF4-NEXT: [[TMP58]] = call @llvm.vector.extract.nxv16i1.nxv64i1( [[ACTIVE_LANE_MASK_NEXT]], i64 48) -; CHECK-UF4-NEXT: [[TMP57]] = call @llvm.vector.extract.nxv16i1.nxv64i1( [[ACTIVE_LANE_MASK_NEXT]], i64 32) -; CHECK-UF4-NEXT: [[TMP56]] = call @llvm.vector.extract.nxv16i1.nxv64i1( [[ACTIVE_LANE_MASK_NEXT]], i64 16) -; CHECK-UF4-NEXT: [[TMP55]] = call @llvm.vector.extract.nxv16i1.nxv64i1( [[ACTIVE_LANE_MASK_NEXT]], i64 0) -; CHECK-UF4-NEXT: [[TMP59:%.*]] = extractelement [[TMP55]], i64 0 -; CHECK-UF4-NEXT: [[TMP60:%.*]] = xor i1 [[TMP59]], true -; CHECK-UF4-NEXT: br i1 [[TMP60]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] +; CHECK-UF4-NEXT: [[TMP21]] = call @llvm.vector.extract.nxv16i1.nxv64i1( [[ACTIVE_LANE_MASK_NEXT]], i64 0) +; CHECK-UF4-NEXT: [[TMP22]] = call @llvm.vector.extract.nxv16i1.nxv64i1( [[ACTIVE_LANE_MASK_NEXT]], i64 16) +; CHECK-UF4-NEXT: [[TMP23]] = call @llvm.vector.extract.nxv16i1.nxv64i1( [[ACTIVE_LANE_MASK_NEXT]], i64 32) +; CHECK-UF4-NEXT: [[TMP24]] = call @llvm.vector.extract.nxv16i1.nxv64i1( [[ACTIVE_LANE_MASK_NEXT]], i64 48) +; CHECK-UF4-NEXT: [[TMP25:%.*]] = extractelement [[TMP21]], i64 0 +; CHECK-UF4-NEXT: [[TMP26:%.*]] = xor i1 [[TMP25]], true +; CHECK-UF4-NEXT: br i1 [[TMP26]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] ; CHECK-UF4: middle.block: ; -; CHECK-TF-LABEL: define void @scalable_wide_active_lane_mask( -; CHECK-TF-SAME: ptr noalias [[DST:%.*]], ptr readonly [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] { -; CHECK-TF-NEXT: entry: -; CHECK-TF-NEXT: br label [[VECTOR_PH:%.*]] -; CHECK-TF: vector.ph: -; CHECK-TF-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-TF-NEXT: [[TMP10:%.*]] = shl nuw i64 [[TMP0]], 4 -; CHECK-TF-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP10]], 1 -; CHECK-TF-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call @llvm.get.active.lane.mask.nxv32i1.i64(i64 0, i64 [[N]]) -; CHECK-TF-NEXT: [[TMP7:%.*]] = call @llvm.vector.extract.nxv16i1.nxv32i1( [[ACTIVE_LANE_MASK_ENTRY]], i64 16) -; CHECK-TF-NEXT: [[TMP8:%.*]] = call @llvm.vector.extract.nxv16i1.nxv32i1( [[ACTIVE_LANE_MASK_ENTRY]], i64 0) -; CHECK-TF-NEXT: br label [[VECTOR_BODY:%.*]] -; CHECK-TF: vector.body: -; CHECK-TF-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-TF-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi [ [[TMP8]], [[VECTOR_PH]] ], [ [[TMP20:%.*]], [[VECTOR_BODY]] ] -; CHECK-TF-NEXT: [[ACTIVE_LANE_MASK1:%.*]] = phi [ [[TMP7]], [[VECTOR_PH]] ], [ [[TMP19:%.*]], [[VECTOR_BODY]] ] -; CHECK-TF-NEXT: [[TMP9:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 [[INDEX]] -; CHECK-TF-NEXT: [[TMP12:%.*]] = getelementptr inbounds i8, ptr [[TMP9]], i64 [[TMP10]] -; CHECK-TF-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP9]], [[ACTIVE_LANE_MASK]], poison) -; CHECK-TF-NEXT: [[WIDE_MASKED_LOAD2:%.*]] = call @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP12]], [[ACTIVE_LANE_MASK1]], poison) -; CHECK-TF-NEXT: [[TMP13:%.*]] = mul [[WIDE_MASKED_LOAD]], splat (i8 3) -; CHECK-TF-NEXT: [[TMP14:%.*]] = mul [[WIDE_MASKED_LOAD2]], splat (i8 3) -; CHECK-TF-NEXT: [[TMP15:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[INDEX]] -; CHECK-TF-NEXT: [[TMP18:%.*]] = getelementptr inbounds i8, ptr [[TMP15]], i64 [[TMP10]] -; CHECK-TF-NEXT: call void @llvm.masked.store.nxv16i8.p0( [[TMP13]], ptr align 1 [[TMP15]], [[ACTIVE_LANE_MASK]]) -; CHECK-TF-NEXT: call void @llvm.masked.store.nxv16i8.p0( [[TMP14]], ptr align 1 [[TMP18]], [[ACTIVE_LANE_MASK1]]) -; CHECK-TF-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP1]] -; CHECK-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT:%.*]] = call @llvm.get.active.lane.mask.nxv32i1.i64(i64 [[INDEX_NEXT]], i64 [[N]]) -; CHECK-TF-NEXT: [[TMP19]] = call @llvm.vector.extract.nxv16i1.nxv32i1( [[ACTIVE_LANE_MASK_NEXT]], i64 16) -; CHECK-TF-NEXT: [[TMP20]] = call @llvm.vector.extract.nxv16i1.nxv32i1( [[ACTIVE_LANE_MASK_NEXT]], i64 0) -; CHECK-TF-NEXT: [[TMP21:%.*]] = extractelement [[TMP20]], i64 0 -; CHECK-TF-NEXT: [[TMP22:%.*]] = xor i1 [[TMP21]], true -; CHECK-TF-NEXT: br i1 [[TMP22]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] -; CHECK-TF: middle.block: -; entry: br label %for.body @@ -142,123 +106,85 @@ define void @scalable_wide_active_lane_mask_double(ptr noalias %dst, ptr readonl ; CHECK-UF1-SAME: ptr noalias [[DST:%.*]], ptr readonly [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0]] { ; CHECK-UF1-NEXT: entry: ; CHECK-UF1-NEXT: [[CMP6:%.*]] = icmp sgt i64 [[N]], 0 -; CHECK-UF1-NEXT: br i1 [[CMP6]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]] +; CHECK-UF1-NEXT: br i1 [[CMP6]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_END:%.*]] ; CHECK-UF1: for.body.preheader: ; CHECK-UF1-NEXT: br label [[VECTOR_PH:%.*]] ; CHECK-UF1: vector.ph: -; CHECK-UF1-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-UF1-NEXT: [[TMP9:%.*]] = shl nuw i64 [[TMP2]], 1 +; CHECK-UF1-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-UF1-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1 ; CHECK-UF1-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 [[N]]) ; CHECK-UF1-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK-UF1: vector.body: ; CHECK-UF1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] ; CHECK-UF1-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-UF1-NEXT: [[TMP5:%.*]] = getelementptr inbounds double, ptr [[SRC]], i64 [[INDEX]] -; CHECK-UF1-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call @llvm.masked.load.nxv2f64.p0(ptr align 8 [[TMP5]], [[ACTIVE_LANE_MASK]], poison) -; CHECK-UF1-NEXT: [[TMP3:%.*]] = fmul [[WIDE_MASKED_LOAD]], splat (double 3.000000e+00) -; CHECK-UF1-NEXT: [[TMP8:%.*]] = getelementptr inbounds double, ptr [[DST]], i64 [[INDEX]] -; CHECK-UF1-NEXT: call void @llvm.masked.store.nxv2f64.p0( [[TMP3]], ptr align 8 [[TMP8]], [[ACTIVE_LANE_MASK]]) -; CHECK-UF1-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP9]] +; CHECK-UF1-NEXT: [[TMP3:%.*]] = getelementptr inbounds double, ptr [[SRC]], i64 [[INDEX]] +; CHECK-UF1-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call @llvm.masked.load.nxv2f64.p0(ptr align 8 [[TMP3]], [[ACTIVE_LANE_MASK]], poison) +; CHECK-UF1-NEXT: [[TMP4:%.*]] = fmul [[WIDE_MASKED_LOAD]], splat (double 3.000000e+00) +; CHECK-UF1-NEXT: [[TMP5:%.*]] = getelementptr inbounds double, ptr [[DST]], i64 [[INDEX]] +; CHECK-UF1-NEXT: call void @llvm.masked.store.nxv2f64.p0( [[TMP4]], ptr align 8 [[TMP5]], [[ACTIVE_LANE_MASK]]) +; CHECK-UF1-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP1]] ; CHECK-UF1-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX_NEXT]], i64 [[N]]) -; CHECK-UF1-NEXT: [[TMP7:%.*]] = extractelement [[ACTIVE_LANE_MASK_NEXT]], i64 0 -; CHECK-UF1-NEXT: [[TMP6:%.*]] = xor i1 [[TMP7]], true -; CHECK-UF1-NEXT: br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]] +; CHECK-UF1-NEXT: [[TMP6:%.*]] = extractelement [[ACTIVE_LANE_MASK_NEXT]], i64 0 +; CHECK-UF1-NEXT: [[TMP7:%.*]] = xor i1 [[TMP6]], true +; CHECK-UF1-NEXT: br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]] ; CHECK-UF1: middle.block: ; ; CHECK-UF4-LABEL: define void @scalable_wide_active_lane_mask_double( ; CHECK-UF4-SAME: ptr noalias [[DST:%.*]], ptr readonly [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0]] { ; CHECK-UF4-NEXT: entry: ; CHECK-UF4-NEXT: [[CMP6:%.*]] = icmp sgt i64 [[N]], 0 -; CHECK-UF4-NEXT: br i1 [[CMP6]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]] +; CHECK-UF4-NEXT: br i1 [[CMP6]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_END:%.*]] ; CHECK-UF4: for.body.preheader: ; CHECK-UF4-NEXT: br label [[VECTOR_PH:%.*]] ; CHECK-UF4: vector.ph: -; CHECK-UF4-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-UF4-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP2]], 1 -; CHECK-UF4-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP1]], 2 +; CHECK-UF4-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-UF4-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1 +; CHECK-UF4-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 2 ; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 [[N]]) -; CHECK-UF4-NEXT: [[TMP14:%.*]] = call @llvm.vector.extract.nxv2i1.nxv8i1( [[ACTIVE_LANE_MASK_ENTRY]], i64 6) -; CHECK-UF4-NEXT: [[TMP13:%.*]] = call @llvm.vector.extract.nxv2i1.nxv8i1( [[ACTIVE_LANE_MASK_ENTRY]], i64 4) -; CHECK-UF4-NEXT: [[TMP12:%.*]] = call @llvm.vector.extract.nxv2i1.nxv8i1( [[ACTIVE_LANE_MASK_ENTRY]], i64 2) -; CHECK-UF4-NEXT: [[TMP11:%.*]] = call @llvm.vector.extract.nxv2i1.nxv8i1( [[ACTIVE_LANE_MASK_ENTRY]], i64 0) +; CHECK-UF4-NEXT: [[TMP3:%.*]] = call @llvm.vector.extract.nxv2i1.nxv8i1( [[ACTIVE_LANE_MASK_ENTRY]], i64 0) +; CHECK-UF4-NEXT: [[EXTRACT_ENTRY_ALM_PART1:%.*]] = call @llvm.vector.extract.nxv2i1.nxv8i1( [[ACTIVE_LANE_MASK_ENTRY]], i64 2) +; CHECK-UF4-NEXT: [[EXTRACT_ENTRY_ALM_PART2:%.*]] = call @llvm.vector.extract.nxv2i1.nxv8i1( [[ACTIVE_LANE_MASK_ENTRY]], i64 4) +; CHECK-UF4-NEXT: [[EXTRACT_ENTRY_ALM_PART3:%.*]] = call @llvm.vector.extract.nxv2i1.nxv8i1( [[ACTIVE_LANE_MASK_ENTRY]], i64 6) ; CHECK-UF4-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK-UF4: vector.body: ; CHECK-UF4-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi [ [[TMP11]], [[VECTOR_PH]] ], [ [[TMP50:%.*]], [[VECTOR_BODY]] ] -; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK6:%.*]] = phi [ [[TMP12]], [[VECTOR_PH]] ], [ [[TMP51:%.*]], [[VECTOR_BODY]] ] -; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK7:%.*]] = phi [ [[TMP13]], [[VECTOR_PH]] ], [ [[TMP52:%.*]], [[VECTOR_BODY]] ] -; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK8:%.*]] = phi [ [[TMP14]], [[VECTOR_PH]] ], [ [[TMP53:%.*]], [[VECTOR_BODY]] ] -; CHECK-UF4-NEXT: [[TMP15:%.*]] = getelementptr inbounds double, ptr [[SRC]], i64 [[INDEX]] -; CHECK-UF4-NEXT: [[TMP21:%.*]] = shl nuw nsw i64 [[TMP1]], 1 -; CHECK-UF4-NEXT: [[TMP24:%.*]] = mul nuw nsw i64 [[TMP1]], 3 -; CHECK-UF4-NEXT: [[TMP29:%.*]] = getelementptr inbounds double, ptr [[TMP15]], i64 [[TMP1]] -; CHECK-UF4-NEXT: [[TMP22:%.*]] = getelementptr inbounds double, ptr [[TMP15]], i64 [[TMP21]] -; CHECK-UF4-NEXT: [[TMP25:%.*]] = getelementptr inbounds double, ptr [[TMP15]], i64 [[TMP24]] -; CHECK-UF4-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call @llvm.masked.load.nxv2f64.p0(ptr align 8 [[TMP15]], [[ACTIVE_LANE_MASK]], poison) -; CHECK-UF4-NEXT: [[WIDE_MASKED_LOAD9:%.*]] = call @llvm.masked.load.nxv2f64.p0(ptr align 8 [[TMP29]], [[ACTIVE_LANE_MASK6]], poison) -; CHECK-UF4-NEXT: [[WIDE_MASKED_LOAD10:%.*]] = call @llvm.masked.load.nxv2f64.p0(ptr align 8 [[TMP22]], [[ACTIVE_LANE_MASK7]], poison) -; CHECK-UF4-NEXT: [[WIDE_MASKED_LOAD11:%.*]] = call @llvm.masked.load.nxv2f64.p0(ptr align 8 [[TMP25]], [[ACTIVE_LANE_MASK8]], poison) -; CHECK-UF4-NEXT: [[TMP16:%.*]] = fmul [[WIDE_MASKED_LOAD]], splat (double 3.000000e+00) -; CHECK-UF4-NEXT: [[TMP17:%.*]] = fmul [[WIDE_MASKED_LOAD9]], splat (double 3.000000e+00) -; CHECK-UF4-NEXT: [[TMP18:%.*]] = fmul [[WIDE_MASKED_LOAD10]], splat (double 3.000000e+00) -; CHECK-UF4-NEXT: [[TMP19:%.*]] = fmul [[WIDE_MASKED_LOAD11]], splat (double 3.000000e+00) -; CHECK-UF4-NEXT: [[TMP30:%.*]] = getelementptr inbounds double, ptr [[DST]], i64 [[INDEX]] -; CHECK-UF4-NEXT: [[TMP34:%.*]] = getelementptr inbounds double, ptr [[TMP30]], i64 [[TMP1]] -; CHECK-UF4-NEXT: [[TMP37:%.*]] = getelementptr inbounds double, ptr [[TMP30]], i64 [[TMP21]] -; CHECK-UF4-NEXT: [[TMP40:%.*]] = getelementptr inbounds double, ptr [[TMP30]], i64 [[TMP24]] -; CHECK-UF4-NEXT: call void @llvm.masked.store.nxv2f64.p0( [[TMP16]], ptr align 8 [[TMP30]], [[ACTIVE_LANE_MASK]]) -; CHECK-UF4-NEXT: call void @llvm.masked.store.nxv2f64.p0( [[TMP17]], ptr align 8 [[TMP34]], [[ACTIVE_LANE_MASK6]]) -; CHECK-UF4-NEXT: call void @llvm.masked.store.nxv2f64.p0( [[TMP18]], ptr align 8 [[TMP37]], [[ACTIVE_LANE_MASK7]]) -; CHECK-UF4-NEXT: call void @llvm.masked.store.nxv2f64.p0( [[TMP19]], ptr align 8 [[TMP40]], [[ACTIVE_LANE_MASK8]]) -; CHECK-UF4-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP3]] +; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi [ [[TMP3]], [[VECTOR_PH]] ], [ [[TMP21:%.*]], [[VECTOR_BODY]] ] +; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK1:%.*]] = phi [ [[EXTRACT_ENTRY_ALM_PART1]], [[VECTOR_PH]] ], [ [[TMP22:%.*]], [[VECTOR_BODY]] ] +; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK2:%.*]] = phi [ [[EXTRACT_ENTRY_ALM_PART2]], [[VECTOR_PH]] ], [ [[TMP23:%.*]], [[VECTOR_BODY]] ] +; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK3:%.*]] = phi [ [[EXTRACT_ENTRY_ALM_PART3]], [[VECTOR_PH]] ], [ [[TMP24:%.*]], [[VECTOR_BODY]] ] +; CHECK-UF4-NEXT: [[TMP7:%.*]] = getelementptr inbounds double, ptr [[SRC]], i64 [[INDEX]] +; CHECK-UF4-NEXT: [[TMP8:%.*]] = shl nuw nsw i64 [[TMP1]], 1 +; CHECK-UF4-NEXT: [[TMP9:%.*]] = mul nuw nsw i64 [[TMP1]], 3 +; CHECK-UF4-NEXT: [[TMP10:%.*]] = getelementptr inbounds double, ptr [[TMP7]], i64 [[TMP1]] +; CHECK-UF4-NEXT: [[TMP11:%.*]] = getelementptr inbounds double, ptr [[TMP7]], i64 [[TMP8]] +; CHECK-UF4-NEXT: [[TMP12:%.*]] = getelementptr inbounds double, ptr [[TMP7]], i64 [[TMP9]] +; CHECK-UF4-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call @llvm.masked.load.nxv2f64.p0(ptr align 8 [[TMP7]], [[ACTIVE_LANE_MASK]], poison) +; CHECK-UF4-NEXT: [[WIDE_MASKED_LOAD4:%.*]] = call @llvm.masked.load.nxv2f64.p0(ptr align 8 [[TMP10]], [[ACTIVE_LANE_MASK1]], poison) +; CHECK-UF4-NEXT: [[WIDE_MASKED_LOAD5:%.*]] = call @llvm.masked.load.nxv2f64.p0(ptr align 8 [[TMP11]], [[ACTIVE_LANE_MASK2]], poison) +; CHECK-UF4-NEXT: [[WIDE_MASKED_LOAD6:%.*]] = call @llvm.masked.load.nxv2f64.p0(ptr align 8 [[TMP12]], [[ACTIVE_LANE_MASK3]], poison) +; CHECK-UF4-NEXT: [[TMP13:%.*]] = fmul [[WIDE_MASKED_LOAD]], splat (double 3.000000e+00) +; CHECK-UF4-NEXT: [[TMP14:%.*]] = fmul [[WIDE_MASKED_LOAD4]], splat (double 3.000000e+00) +; CHECK-UF4-NEXT: [[TMP15:%.*]] = fmul [[WIDE_MASKED_LOAD5]], splat (double 3.000000e+00) +; CHECK-UF4-NEXT: [[TMP16:%.*]] = fmul [[WIDE_MASKED_LOAD6]], splat (double 3.000000e+00) +; CHECK-UF4-NEXT: [[TMP17:%.*]] = getelementptr inbounds double, ptr [[DST]], i64 [[INDEX]] +; CHECK-UF4-NEXT: [[TMP18:%.*]] = getelementptr inbounds double, ptr [[TMP17]], i64 [[TMP1]] +; CHECK-UF4-NEXT: [[TMP19:%.*]] = getelementptr inbounds double, ptr [[TMP17]], i64 [[TMP8]] +; CHECK-UF4-NEXT: [[TMP20:%.*]] = getelementptr inbounds double, ptr [[TMP17]], i64 [[TMP9]] +; CHECK-UF4-NEXT: call void @llvm.masked.store.nxv2f64.p0( [[TMP13]], ptr align 8 [[TMP17]], [[ACTIVE_LANE_MASK]]) +; CHECK-UF4-NEXT: call void @llvm.masked.store.nxv2f64.p0( [[TMP14]], ptr align 8 [[TMP18]], [[ACTIVE_LANE_MASK1]]) +; CHECK-UF4-NEXT: call void @llvm.masked.store.nxv2f64.p0( [[TMP15]], ptr align 8 [[TMP19]], [[ACTIVE_LANE_MASK2]]) +; CHECK-UF4-NEXT: call void @llvm.masked.store.nxv2f64.p0( [[TMP16]], ptr align 8 [[TMP20]], [[ACTIVE_LANE_MASK3]]) +; CHECK-UF4-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP2]] ; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK_NEXT:%.*]] = call @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[INDEX_NEXT]], i64 [[N]]) -; CHECK-UF4-NEXT: [[TMP53]] = call @llvm.vector.extract.nxv2i1.nxv8i1( [[ACTIVE_LANE_MASK_NEXT]], i64 6) -; CHECK-UF4-NEXT: [[TMP52]] = call @llvm.vector.extract.nxv2i1.nxv8i1( [[ACTIVE_LANE_MASK_NEXT]], i64 4) -; CHECK-UF4-NEXT: [[TMP51]] = call @llvm.vector.extract.nxv2i1.nxv8i1( [[ACTIVE_LANE_MASK_NEXT]], i64 2) -; CHECK-UF4-NEXT: [[TMP50]] = call @llvm.vector.extract.nxv2i1.nxv8i1( [[ACTIVE_LANE_MASK_NEXT]], i64 0) -; CHECK-UF4-NEXT: [[TMP54:%.*]] = extractelement [[TMP50]], i64 0 -; CHECK-UF4-NEXT: [[TMP55:%.*]] = xor i1 [[TMP54]], true -; CHECK-UF4-NEXT: br i1 [[TMP55]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]] +; CHECK-UF4-NEXT: [[TMP21]] = call @llvm.vector.extract.nxv2i1.nxv8i1( [[ACTIVE_LANE_MASK_NEXT]], i64 0) +; CHECK-UF4-NEXT: [[TMP22]] = call @llvm.vector.extract.nxv2i1.nxv8i1( [[ACTIVE_LANE_MASK_NEXT]], i64 2) +; CHECK-UF4-NEXT: [[TMP23]] = call @llvm.vector.extract.nxv2i1.nxv8i1( [[ACTIVE_LANE_MASK_NEXT]], i64 4) +; CHECK-UF4-NEXT: [[TMP24]] = call @llvm.vector.extract.nxv2i1.nxv8i1( [[ACTIVE_LANE_MASK_NEXT]], i64 6) +; CHECK-UF4-NEXT: [[TMP25:%.*]] = extractelement [[TMP21]], i64 0 +; CHECK-UF4-NEXT: [[TMP26:%.*]] = xor i1 [[TMP25]], true +; CHECK-UF4-NEXT: br i1 [[TMP26]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]] ; CHECK-UF4: middle.block: ; -; CHECK-TF-LABEL: define void @scalable_wide_active_lane_mask_double( -; CHECK-TF-SAME: ptr noalias [[DST:%.*]], ptr readonly [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0]] { -; CHECK-TF-NEXT: entry: -; CHECK-TF-NEXT: [[CMP6:%.*]] = icmp sgt i64 [[N]], 0 -; CHECK-TF-NEXT: br i1 [[CMP6]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_END:%.*]] -; CHECK-TF: for.body.preheader: -; CHECK-TF-NEXT: br label [[VECTOR_PH:%.*]] -; CHECK-TF: vector.ph: -; CHECK-TF-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-TF-NEXT: [[TMP10:%.*]] = shl nuw i64 [[TMP0]], 1 -; CHECK-TF-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP10]], 1 -; CHECK-TF-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[N]]) -; CHECK-TF-NEXT: [[TMP7:%.*]] = call @llvm.vector.extract.nxv2i1.nxv4i1( [[ACTIVE_LANE_MASK_ENTRY]], i64 2) -; CHECK-TF-NEXT: [[TMP8:%.*]] = call @llvm.vector.extract.nxv2i1.nxv4i1( [[ACTIVE_LANE_MASK_ENTRY]], i64 0) -; CHECK-TF-NEXT: br label [[VECTOR_BODY:%.*]] -; CHECK-TF: vector.body: -; CHECK-TF-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-TF-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi [ [[TMP8]], [[VECTOR_PH]] ], [ [[TMP20:%.*]], [[VECTOR_BODY]] ] -; CHECK-TF-NEXT: [[ACTIVE_LANE_MASK1:%.*]] = phi [ [[TMP7]], [[VECTOR_PH]] ], [ [[TMP19:%.*]], [[VECTOR_BODY]] ] -; CHECK-TF-NEXT: [[TMP9:%.*]] = getelementptr inbounds double, ptr [[SRC]], i64 [[INDEX]] -; CHECK-TF-NEXT: [[TMP12:%.*]] = getelementptr inbounds double, ptr [[TMP9]], i64 [[TMP10]] -; CHECK-TF-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call @llvm.masked.load.nxv2f64.p0(ptr align 8 [[TMP9]], [[ACTIVE_LANE_MASK]], poison) -; CHECK-TF-NEXT: [[WIDE_MASKED_LOAD2:%.*]] = call @llvm.masked.load.nxv2f64.p0(ptr align 8 [[TMP12]], [[ACTIVE_LANE_MASK1]], poison) -; CHECK-TF-NEXT: [[TMP13:%.*]] = fmul [[WIDE_MASKED_LOAD]], splat (double 3.000000e+00) -; CHECK-TF-NEXT: [[TMP14:%.*]] = fmul [[WIDE_MASKED_LOAD2]], splat (double 3.000000e+00) -; CHECK-TF-NEXT: [[TMP15:%.*]] = getelementptr inbounds double, ptr [[DST]], i64 [[INDEX]] -; CHECK-TF-NEXT: [[TMP18:%.*]] = getelementptr inbounds double, ptr [[TMP15]], i64 [[TMP10]] -; CHECK-TF-NEXT: call void @llvm.masked.store.nxv2f64.p0( [[TMP13]], ptr align 8 [[TMP15]], [[ACTIVE_LANE_MASK]]) -; CHECK-TF-NEXT: call void @llvm.masked.store.nxv2f64.p0( [[TMP14]], ptr align 8 [[TMP18]], [[ACTIVE_LANE_MASK1]]) -; CHECK-TF-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP1]] -; CHECK-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT:%.*]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT]], i64 [[N]]) -; CHECK-TF-NEXT: [[TMP19]] = call @llvm.vector.extract.nxv2i1.nxv4i1( [[ACTIVE_LANE_MASK_NEXT]], i64 2) -; CHECK-TF-NEXT: [[TMP20]] = call @llvm.vector.extract.nxv2i1.nxv4i1( [[ACTIVE_LANE_MASK_NEXT]], i64 0) -; CHECK-TF-NEXT: [[TMP21:%.*]] = extractelement [[TMP20]], i64 0 -; CHECK-TF-NEXT: [[TMP22:%.*]] = xor i1 [[TMP21]], true -; CHECK-TF-NEXT: br i1 [[TMP22]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]] -; CHECK-TF: middle.block: -; entry: %cmp6 = icmp sgt i64 %n, 0 br i1 %cmp6, label %for.body, label %for.end @@ -279,4 +205,3 @@ for.end: } attributes #0 = { nounwind vscale_range(1,16) "target-features"="+sve2p1" } - diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/uniform-args-call-variants.ll b/llvm/test/Transforms/LoopVectorize/AArch64/uniform-args-call-variants.ll index 094ce721f20c6..91ff96cd7f2d6 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/uniform-args-call-variants.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/uniform-args-call-variants.ll @@ -42,13 +42,14 @@ define void @test_uniform(ptr noalias %dst, ptr readonly %src, i64 %uniform , i6 ; INTERLEAVE-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64() ; INTERLEAVE-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1 ; INTERLEAVE-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 1 -; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 [[N]]) -; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY1:%.*]] = call @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP1]], i64 [[N]]) +; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[N]]) +; INTERLEAVE-NEXT: [[EXTRACT_ENTRY_ALM_PART:%.*]] = call @llvm.vector.extract.nxv2i1.nxv4i1( [[ACTIVE_LANE_MASK_ENTRY]], i64 0) +; INTERLEAVE-NEXT: [[EXTRACT_ENTRY_ALM_PART1:%.*]] = call @llvm.vector.extract.nxv2i1.nxv4i1( [[ACTIVE_LANE_MASK_ENTRY]], i64 2) ; INTERLEAVE-NEXT: br label [[VECTOR_BODY:%.*]] ; INTERLEAVE: vector.body: ; INTERLEAVE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ] -; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK2:%.*]] = phi [ [[ACTIVE_LANE_MASK_ENTRY1]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT4:%.*]], [[VECTOR_BODY]] ] +; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi [ [[EXTRACT_ENTRY_ALM_PART]], [[VECTOR_PH]] ], [ [[EXTRACT_NEXT_ALM_PART:%.*]], [[VECTOR_BODY]] ] +; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK2:%.*]] = phi [ [[EXTRACT_ENTRY_ALM_PART1]], [[VECTOR_PH]] ], [ [[EXTRACT_NEXT_ALM_PART4:%.*]], [[VECTOR_BODY]] ] ; INTERLEAVE-NEXT: [[TMP3:%.*]] = getelementptr double, ptr [[SRC]], i64 [[INDEX]] ; INTERLEAVE-NEXT: [[TMP4:%.*]] = getelementptr double, ptr [[TMP3]], i64 [[TMP1]] ; INTERLEAVE-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call @llvm.masked.load.nxv2f64.p0(ptr align 8 [[TMP3]], [[ACTIVE_LANE_MASK]], poison) @@ -60,12 +61,12 @@ define void @test_uniform(ptr noalias %dst, ptr readonly %src, i64 %uniform , i6 ; INTERLEAVE-NEXT: call void @llvm.masked.store.nxv2f64.p0( [[TMP5]], ptr align 8 [[TMP7]], [[ACTIVE_LANE_MASK]]) ; INTERLEAVE-NEXT: call void @llvm.masked.store.nxv2f64.p0( [[TMP6]], ptr align 8 [[TMP8]], [[ACTIVE_LANE_MASK2]]) ; INTERLEAVE-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP2]] -; INTERLEAVE-NEXT: [[TMP9:%.*]] = add i64 [[INDEX_NEXT]], [[TMP1]] -; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX_NEXT]], i64 [[N]]) -; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT4]] = call @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP9]], i64 [[N]]) -; INTERLEAVE-NEXT: [[TMP10:%.*]] = extractelement [[ACTIVE_LANE_MASK_NEXT]], i64 0 -; INTERLEAVE-NEXT: [[TMP11:%.*]] = xor i1 [[TMP10]], true -; INTERLEAVE-NEXT: br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] +; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT:%.*]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT]], i64 [[N]]) +; INTERLEAVE-NEXT: [[EXTRACT_NEXT_ALM_PART]] = call @llvm.vector.extract.nxv2i1.nxv4i1( [[ACTIVE_LANE_MASK_NEXT]], i64 0) +; INTERLEAVE-NEXT: [[EXTRACT_NEXT_ALM_PART4]] = call @llvm.vector.extract.nxv2i1.nxv4i1( [[ACTIVE_LANE_MASK_NEXT]], i64 2) +; INTERLEAVE-NEXT: [[TMP9:%.*]] = extractelement [[EXTRACT_NEXT_ALM_PART]], i64 0 +; INTERLEAVE-NEXT: [[TMP10:%.*]] = xor i1 [[TMP9]], true +; INTERLEAVE-NEXT: br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] ; INTERLEAVE: middle.block: ; INTERLEAVE-NEXT: br label [[FOR_COND_CLEANUP:%.*]] ; INTERLEAVE: for.cond.cleanup: @@ -125,13 +126,14 @@ define void @test_uniform_smaller_scalar(ptr noalias %dst, ptr readonly %src, i3 ; INTERLEAVE-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64() ; INTERLEAVE-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1 ; INTERLEAVE-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 1 -; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 [[N]]) -; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY1:%.*]] = call @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP1]], i64 [[N]]) +; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[N]]) +; INTERLEAVE-NEXT: [[EXTRACT_ENTRY_ALM_PART:%.*]] = call @llvm.vector.extract.nxv2i1.nxv4i1( [[ACTIVE_LANE_MASK_ENTRY]], i64 0) +; INTERLEAVE-NEXT: [[EXTRACT_ENTRY_ALM_PART1:%.*]] = call @llvm.vector.extract.nxv2i1.nxv4i1( [[ACTIVE_LANE_MASK_ENTRY]], i64 2) ; INTERLEAVE-NEXT: br label [[VECTOR_BODY:%.*]] ; INTERLEAVE: vector.body: ; INTERLEAVE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ] -; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK2:%.*]] = phi [ [[ACTIVE_LANE_MASK_ENTRY1]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT4:%.*]], [[VECTOR_BODY]] ] +; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi [ [[EXTRACT_ENTRY_ALM_PART]], [[VECTOR_PH]] ], [ [[EXTRACT_NEXT_ALM_PART:%.*]], [[VECTOR_BODY]] ] +; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK2:%.*]] = phi [ [[EXTRACT_ENTRY_ALM_PART1]], [[VECTOR_PH]] ], [ [[EXTRACT_NEXT_ALM_PART4:%.*]], [[VECTOR_BODY]] ] ; INTERLEAVE-NEXT: [[TMP3:%.*]] = getelementptr double, ptr [[SRC]], i64 [[INDEX]] ; INTERLEAVE-NEXT: [[TMP4:%.*]] = getelementptr double, ptr [[TMP3]], i64 [[TMP1]] ; INTERLEAVE-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call @llvm.masked.load.nxv2f64.p0(ptr align 8 [[TMP3]], [[ACTIVE_LANE_MASK]], poison) @@ -143,12 +145,12 @@ define void @test_uniform_smaller_scalar(ptr noalias %dst, ptr readonly %src, i3 ; INTERLEAVE-NEXT: call void @llvm.masked.store.nxv2f64.p0( [[TMP5]], ptr align 8 [[TMP7]], [[ACTIVE_LANE_MASK]]) ; INTERLEAVE-NEXT: call void @llvm.masked.store.nxv2f64.p0( [[TMP6]], ptr align 8 [[TMP8]], [[ACTIVE_LANE_MASK2]]) ; INTERLEAVE-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP2]] -; INTERLEAVE-NEXT: [[TMP9:%.*]] = add i64 [[INDEX_NEXT]], [[TMP1]] -; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX_NEXT]], i64 [[N]]) -; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT4]] = call @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP9]], i64 [[N]]) -; INTERLEAVE-NEXT: [[TMP10:%.*]] = extractelement [[ACTIVE_LANE_MASK_NEXT]], i64 0 -; INTERLEAVE-NEXT: [[TMP11:%.*]] = xor i1 [[TMP10]], true -; INTERLEAVE-NEXT: br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]] +; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT:%.*]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT]], i64 [[N]]) +; INTERLEAVE-NEXT: [[EXTRACT_NEXT_ALM_PART]] = call @llvm.vector.extract.nxv2i1.nxv4i1( [[ACTIVE_LANE_MASK_NEXT]], i64 0) +; INTERLEAVE-NEXT: [[EXTRACT_NEXT_ALM_PART4]] = call @llvm.vector.extract.nxv2i1.nxv4i1( [[ACTIVE_LANE_MASK_NEXT]], i64 2) +; INTERLEAVE-NEXT: [[TMP9:%.*]] = extractelement [[EXTRACT_NEXT_ALM_PART]], i64 0 +; INTERLEAVE-NEXT: [[TMP10:%.*]] = xor i1 [[TMP9]], true +; INTERLEAVE-NEXT: br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]] ; INTERLEAVE: middle.block: ; INTERLEAVE-NEXT: br label [[FOR_COND_CLEANUP:%.*]] ; INTERLEAVE: for.cond.cleanup: @@ -196,38 +198,38 @@ define void @test_uniform_not_invariant(ptr noalias %dst, ptr readonly %src, i64 ; INTERLEAVE-NEXT: entry: ; INTERLEAVE-NEXT: br label [[VECTOR_PH:%.*]] ; INTERLEAVE: vector.ph: -; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = icmp ult i64 0, [[N]] -; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY1:%.*]] = icmp ult i64 1, [[N]] +; INTERLEAVE-NEXT: [[TMP0:%.*]] = icmp ult i64 0, [[N]] +; INTERLEAVE-NEXT: [[TMP1:%.*]] = icmp ult i64 1, [[N]] ; INTERLEAVE-NEXT: br label [[VECTOR_BODY:%.*]] ; INTERLEAVE: vector.body: -; INTERLEAVE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE4:%.*]] ] -; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi i1 [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[PRED_STORE_CONTINUE4]] ] -; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK2:%.*]] = phi i1 [ [[ACTIVE_LANE_MASK_ENTRY1]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT5:%.*]], [[PRED_STORE_CONTINUE4]] ] +; INTERLEAVE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE3:%.*]] ] +; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi i1 [ [[TMP0]], [[VECTOR_PH]] ], [ [[TMP11:%.*]], [[PRED_STORE_CONTINUE3]] ] +; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK1:%.*]] = phi i1 [ [[TMP1]], [[VECTOR_PH]] ], [ [[TMP13:%.*]], [[PRED_STORE_CONTINUE3]] ] ; INTERLEAVE-NEXT: br i1 [[ACTIVE_LANE_MASK]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]] ; INTERLEAVE: pred.store.if: -; INTERLEAVE-NEXT: [[TMP0:%.*]] = getelementptr double, ptr [[SRC]], i64 [[INDEX]] -; INTERLEAVE-NEXT: [[TMP1:%.*]] = load double, ptr [[TMP0]], align 8 -; INTERLEAVE-NEXT: [[TMP2:%.*]] = call double @foo(double [[TMP1]], i64 [[INDEX]]) #[[ATTR4:[0-9]+]] -; INTERLEAVE-NEXT: [[TMP3:%.*]] = getelementptr inbounds double, ptr [[DST]], i64 [[INDEX]] -; INTERLEAVE-NEXT: store double [[TMP2]], ptr [[TMP3]], align 8 +; INTERLEAVE-NEXT: [[TMP2:%.*]] = getelementptr double, ptr [[SRC]], i64 [[INDEX]] +; INTERLEAVE-NEXT: [[TMP3:%.*]] = load double, ptr [[TMP2]], align 8 +; INTERLEAVE-NEXT: [[TMP4:%.*]] = call double @foo(double [[TMP3]], i64 [[INDEX]]) #[[ATTR4:[0-9]+]] +; INTERLEAVE-NEXT: [[TMP5:%.*]] = getelementptr inbounds double, ptr [[DST]], i64 [[INDEX]] +; INTERLEAVE-NEXT: store double [[TMP4]], ptr [[TMP5]], align 8 ; INTERLEAVE-NEXT: br label [[PRED_STORE_CONTINUE]] ; INTERLEAVE: pred.store.continue: -; INTERLEAVE-NEXT: br i1 [[ACTIVE_LANE_MASK2]], label [[PRED_STORE_IF3:%.*]], label [[PRED_STORE_CONTINUE4]] -; INTERLEAVE: pred.store.if3: -; INTERLEAVE-NEXT: [[TMP4:%.*]] = add i64 [[INDEX]], 1 -; INTERLEAVE-NEXT: [[TMP5:%.*]] = getelementptr double, ptr [[SRC]], i64 [[TMP4]] -; INTERLEAVE-NEXT: [[TMP6:%.*]] = load double, ptr [[TMP5]], align 8 -; INTERLEAVE-NEXT: [[TMP7:%.*]] = call double @foo(double [[TMP6]], i64 [[TMP4]]) #[[ATTR4]] -; INTERLEAVE-NEXT: [[TMP8:%.*]] = getelementptr inbounds double, ptr [[DST]], i64 [[TMP4]] -; INTERLEAVE-NEXT: store double [[TMP7]], ptr [[TMP8]], align 8 -; INTERLEAVE-NEXT: br label [[PRED_STORE_CONTINUE4]] -; INTERLEAVE: pred.store.continue4: +; INTERLEAVE-NEXT: br i1 [[ACTIVE_LANE_MASK1]], label [[PRED_STORE_IF2:%.*]], label [[PRED_STORE_CONTINUE3]] +; INTERLEAVE: pred.store.if2: +; INTERLEAVE-NEXT: [[TMP6:%.*]] = add i64 [[INDEX]], 1 +; INTERLEAVE-NEXT: [[TMP7:%.*]] = getelementptr double, ptr [[SRC]], i64 [[TMP6]] +; INTERLEAVE-NEXT: [[TMP8:%.*]] = load double, ptr [[TMP7]], align 8 +; INTERLEAVE-NEXT: [[TMP9:%.*]] = call double @foo(double [[TMP8]], i64 [[TMP6]]) #[[ATTR4]] +; INTERLEAVE-NEXT: [[TMP10:%.*]] = getelementptr inbounds double, ptr [[DST]], i64 [[TMP6]] +; INTERLEAVE-NEXT: store double [[TMP9]], ptr [[TMP10]], align 8 +; INTERLEAVE-NEXT: br label [[PRED_STORE_CONTINUE3]] +; INTERLEAVE: pred.store.continue3: ; INTERLEAVE-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 2 -; INTERLEAVE-NEXT: [[TMP9:%.*]] = add i64 [[INDEX_NEXT]], 1 -; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = icmp ult i64 [[INDEX_NEXT]], [[N]] -; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT5]] = icmp ult i64 [[TMP9]], [[N]] -; INTERLEAVE-NEXT: [[TMP10:%.*]] = xor i1 [[ACTIVE_LANE_MASK_NEXT]], true -; INTERLEAVE-NEXT: br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]] +; INTERLEAVE-NEXT: [[TMP11]] = icmp ult i64 [[INDEX_NEXT]], [[N]] +; INTERLEAVE-NEXT: [[TMP12:%.*]] = add i64 [[INDEX_NEXT]], 1 +; INTERLEAVE-NEXT: [[TMP13]] = icmp ult i64 [[TMP12]], [[N]] +; INTERLEAVE-NEXT: [[TMP14:%.*]] = xor i1 [[TMP11]], true +; INTERLEAVE-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]] ; INTERLEAVE: middle.block: ; INTERLEAVE-NEXT: br label [[FOR_COND_CLEANUP:%.*]] ; INTERLEAVE: for.cond.cleanup: diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/wide-alm-const.ll b/llvm/test/Transforms/LoopVectorize/AArch64/wide-alm-const.ll new file mode 100644 index 0000000000000..7243a4b56358d --- /dev/null +++ b/llvm/test/Transforms/LoopVectorize/AArch64/wide-alm-const.ll @@ -0,0 +1,61 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "^middle.block:" --version 6 +; RUN: opt -S --passes=loop-vectorize -tail-folding-policy=must-fold-tail -force-vector-width=4 -force-vector-interleave=2 < %s | FileCheck %s -check-prefix CHECK-UF4 + +target triple = "aarch64-unknown-linux" + +define void @scalable_wide_active_lane_mask_double(ptr noalias %dst, ptr readonly %src) #0 { +; CHECK-UF4-LABEL: define void @scalable_wide_active_lane_mask_double( +; CHECK-UF4-SAME: ptr noalias [[DST:%.*]], ptr readonly [[SRC:%.*]]) #[[ATTR0:[0-9]+]] { +; CHECK-UF4-NEXT: [[ENTRY:.*:]] +; CHECK-UF4-NEXT: [[CMP6:%.*]] = icmp sgt i64 9, 0 +; CHECK-UF4-NEXT: br i1 [[CMP6]], label %[[FOR_BODY_PREHEADER:.*]], [[FOR_END:label %.*]] +; CHECK-UF4: [[FOR_BODY_PREHEADER]]: +; CHECK-UF4-NEXT: br label %[[VECTOR_PH:.*]] +; CHECK-UF4: [[VECTOR_PH]]: +; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i64(i64 0, i64 9) +; CHECK-UF4-NEXT: [[EXTRACT_ENTRY_ALM_PART:%.*]] = call <4 x i1> @llvm.vector.extract.v4i1.v8i1(<8 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 0) +; CHECK-UF4-NEXT: [[EXTRACT_ENTRY_ALM_PART1:%.*]] = call <4 x i1> @llvm.vector.extract.v4i1.v8i1(<8 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 4) +; CHECK-UF4-NEXT: br label %[[VECTOR_BODY:.*]] +; CHECK-UF4: [[VECTOR_BODY]]: +; CHECK-UF4-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] +; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <4 x i1> [ [[EXTRACT_ENTRY_ALM_PART]], %[[VECTOR_PH]] ], [ [[EXTRACT_NEXT_ALM_PART:%.*]], %[[VECTOR_BODY]] ] +; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK2:%.*]] = phi <4 x i1> [ [[EXTRACT_ENTRY_ALM_PART1]], %[[VECTOR_PH]] ], [ [[EXTRACT_NEXT_ALM_PART4:%.*]], %[[VECTOR_BODY]] ] +; CHECK-UF4-NEXT: [[TMP0:%.*]] = getelementptr inbounds double, ptr [[SRC]], i64 [[INDEX]] +; CHECK-UF4-NEXT: [[TMP1:%.*]] = getelementptr inbounds double, ptr [[TMP0]], i64 4 +; CHECK-UF4-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x double> @llvm.masked.load.v4f64.p0(ptr align 8 [[TMP0]], <4 x i1> [[ACTIVE_LANE_MASK]], <4 x double> poison) +; CHECK-UF4-NEXT: [[WIDE_MASKED_LOAD3:%.*]] = call <4 x double> @llvm.masked.load.v4f64.p0(ptr align 8 [[TMP1]], <4 x i1> [[ACTIVE_LANE_MASK2]], <4 x double> poison) +; CHECK-UF4-NEXT: [[TMP2:%.*]] = fmul <4 x double> [[WIDE_MASKED_LOAD]], splat (double 3.000000e+00) +; CHECK-UF4-NEXT: [[TMP3:%.*]] = fmul <4 x double> [[WIDE_MASKED_LOAD3]], splat (double 3.000000e+00) +; CHECK-UF4-NEXT: [[TMP4:%.*]] = getelementptr inbounds double, ptr [[DST]], i64 [[INDEX]] +; CHECK-UF4-NEXT: [[TMP5:%.*]] = getelementptr inbounds double, ptr [[TMP4]], i64 4 +; CHECK-UF4-NEXT: call void @llvm.masked.store.v4f64.p0(<4 x double> [[TMP2]], ptr align 8 [[TMP4]], <4 x i1> [[ACTIVE_LANE_MASK]]) +; CHECK-UF4-NEXT: call void @llvm.masked.store.v4f64.p0(<4 x double> [[TMP3]], ptr align 8 [[TMP5]], <4 x i1> [[ACTIVE_LANE_MASK2]]) +; CHECK-UF4-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 8 +; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK_NEXT:%.*]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i64(i64 [[INDEX_NEXT]], i64 9) +; CHECK-UF4-NEXT: [[EXTRACT_NEXT_ALM_PART]] = call <4 x i1> @llvm.vector.extract.v4i1.v8i1(<8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0) +; CHECK-UF4-NEXT: [[EXTRACT_NEXT_ALM_PART4]] = call <4 x i1> @llvm.vector.extract.v4i1.v8i1(<8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 4) +; CHECK-UF4-NEXT: [[TMP6:%.*]] = extractelement <4 x i1> [[EXTRACT_NEXT_ALM_PART]], i64 0 +; CHECK-UF4-NEXT: [[TMP7:%.*]] = xor i1 [[TMP6]], true +; CHECK-UF4-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] +; CHECK-UF4: [[MIDDLE_BLOCK]]: +; +entry: + %cmp6 = icmp sgt i64 9, 0 + br i1 %cmp6, label %for.body, label %for.end + +for.body: + %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ] + %arrayidx1 = getelementptr inbounds double, ptr %src, i64 %iv + %ld = load double, ptr %arrayidx1 + %mul = fmul double %ld, 3.000000e+00 + %arrayidx2 = getelementptr inbounds double, ptr %dst, i64 %iv + store double %mul, ptr %arrayidx2 + %iv.next = add nuw nsw i64 %iv, 1 + %exitcond.not = icmp eq i64 %iv.next, 9 + br i1 %exitcond.not, label %for.end, label %for.body + +for.end: + ret void +} + +attributes #0 = { nounwind vscale_range(1,16) "target-features"="+sve2p1" } diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/wide-alm-scalar-vf.ll b/llvm/test/Transforms/LoopVectorize/AArch64/wide-alm-scalar-vf.ll new file mode 100644 index 0000000000000..f6174a0ee37bc --- /dev/null +++ b/llvm/test/Transforms/LoopVectorize/AArch64/wide-alm-scalar-vf.ll @@ -0,0 +1,71 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "^middle.block:" --version 4 +; RUN: opt -S --passes=loop-vectorize -tail-folding-policy=must-fold-tail -force-vector-width=1 -force-vector-interleave=2 < %s | FileCheck %s -check-prefix CHECK-UF4 + +target triple = "aarch64-unknown-linux" + +define void @wide_active_lane_mask_i32_tc(ptr %src, ptr %dst, i32 %n) #0 { +; CHECK-UF4-LABEL: define void @wide_active_lane_mask_i32_tc( +; CHECK-UF4-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]], i32 [[N:%.*]]) #[[ATTR0:[0-9]+]] { +; CHECK-UF4-NEXT: entry: +; CHECK-UF4-NEXT: [[SRC2:%.*]] = ptrtoaddr ptr [[SRC]] to i64 +; CHECK-UF4-NEXT: [[DST1:%.*]] = ptrtoaddr ptr [[DST]] to i64 +; CHECK-UF4-NEXT: br label [[VECTOR_MEMCHECK:%.*]] +; CHECK-UF4: vector.memcheck: +; CHECK-UF4-NEXT: [[TMP0:%.*]] = sub i64 [[DST1]], [[SRC2]] +; CHECK-UF4-NEXT: [[TMP1:%.*]] = sub i64 [[TMP0]], 1 +; CHECK-UF4-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP1]], 15 +; CHECK-UF4-NEXT: br i1 [[DIFF_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] +; CHECK-UF4: vector.ph: +; CHECK-UF4-NEXT: [[TMP2:%.*]] = icmp ult i32 0, [[N]] +; CHECK-UF4-NEXT: [[TMP3:%.*]] = icmp ult i32 1, [[N]] +; CHECK-UF4-NEXT: br label [[VECTOR_BODY:%.*]] +; CHECK-UF4: vector.body: +; CHECK-UF4-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE5:%.*]] ] +; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi i1 [ [[TMP2]], [[VECTOR_PH]] ], [ [[TMP13:%.*]], [[PRED_STORE_CONTINUE5]] ] +; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK3:%.*]] = phi i1 [ [[TMP3]], [[VECTOR_PH]] ], [ [[TMP15:%.*]], [[PRED_STORE_CONTINUE5]] ] +; CHECK-UF4-NEXT: br i1 [[ACTIVE_LANE_MASK]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]] +; CHECK-UF4: pred.store.if: +; CHECK-UF4-NEXT: [[TMP4:%.*]] = getelementptr inbounds double, ptr [[SRC]], i32 [[INDEX]] +; CHECK-UF4-NEXT: [[TMP5:%.*]] = load double, ptr [[TMP4]], align 8 +; CHECK-UF4-NEXT: [[TMP6:%.*]] = getelementptr inbounds double, ptr [[DST]], i32 [[INDEX]] +; CHECK-UF4-NEXT: [[TMP7:%.*]] = fmul double [[TMP5]], 3.000000e+00 +; CHECK-UF4-NEXT: store double [[TMP7]], ptr [[TMP6]], align 8 +; CHECK-UF4-NEXT: br label [[PRED_STORE_CONTINUE]] +; CHECK-UF4: pred.store.continue: +; CHECK-UF4-NEXT: br i1 [[ACTIVE_LANE_MASK3]], label [[PRED_STORE_IF4:%.*]], label [[PRED_STORE_CONTINUE5]] +; CHECK-UF4: pred.store.if4: +; CHECK-UF4-NEXT: [[TMP8:%.*]] = add i32 [[INDEX]], 1 +; CHECK-UF4-NEXT: [[TMP9:%.*]] = getelementptr inbounds double, ptr [[SRC]], i32 [[TMP8]] +; CHECK-UF4-NEXT: [[TMP10:%.*]] = load double, ptr [[TMP9]], align 8 +; CHECK-UF4-NEXT: [[TMP11:%.*]] = getelementptr inbounds double, ptr [[DST]], i32 [[TMP8]] +; CHECK-UF4-NEXT: [[TMP12:%.*]] = fmul double [[TMP10]], 3.000000e+00 +; CHECK-UF4-NEXT: store double [[TMP12]], ptr [[TMP11]], align 8 +; CHECK-UF4-NEXT: br label [[PRED_STORE_CONTINUE5]] +; CHECK-UF4: pred.store.continue5: +; CHECK-UF4-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 2 +; CHECK-UF4-NEXT: [[TMP13]] = icmp ult i32 [[INDEX_NEXT]], [[N]] +; CHECK-UF4-NEXT: [[TMP14:%.*]] = add i32 [[INDEX_NEXT]], 1 +; CHECK-UF4-NEXT: [[TMP15]] = icmp ult i32 [[TMP14]], [[N]] +; CHECK-UF4-NEXT: [[TMP16:%.*]] = xor i1 [[TMP13]], true +; CHECK-UF4-NEXT: br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] +; CHECK-UF4: middle.block: +; +entry: + br label %for.body + +for.body: + %iv = phi i32 [ 0, %entry ], [ %iv.next, %for.body ] + %arrayidx1 = getelementptr inbounds double, ptr %src, i32 %iv + %ld = load double, ptr %arrayidx1 + %mul = fmul double %ld, 3.000000e+00 + %arrayidx2 = getelementptr inbounds double, ptr %dst, i32 %iv + store double %mul, ptr %arrayidx2 + %iv.next = add nuw nsw i32 %iv, 1 + %exitcond.not = icmp eq i32 %iv.next, %n + br i1 %exitcond.not, label %for.end, label %for.body + +for.end: + ret void +} + +attributes #0 = { nounwind vscale_range(1,16) "target-features"="+sve2p1" } diff --git a/llvm/test/Transforms/LoopVectorize/ARM/active-lane-mask.ll b/llvm/test/Transforms/LoopVectorize/ARM/active-lane-mask.ll index 6f82b9301fb45..9a8e204b53e8c 100644 --- a/llvm/test/Transforms/LoopVectorize/ARM/active-lane-mask.ll +++ b/llvm/test/Transforms/LoopVectorize/ARM/active-lane-mask.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5 -; RUN: opt < %s -passes=loop-vectorize -tail-predication=enabled -tail-folding-policy=must-fold-tail -enable-wide-lane-mask -S | FileCheck %s +; RUN: opt < %s -passes=loop-vectorize -tail-predication=enabled -tail-folding-policy=must-fold-tail -S | FileCheck %s target triple = "thumbv8.1m.main-arm-unknown-eabihf" diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/sve-tail-folding-forced.ll b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/sve-tail-folding-forced.ll index c01da392c49ee..d5091fc50eb35 100644 --- a/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/sve-tail-folding-forced.ll +++ b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/sve-tail-folding-forced.ll @@ -18,22 +18,22 @@ target triple = "aarch64-unknown-linux-gnu" ; VPLANS-NEXT: Successor(s): scalar.ph, vector.ph ; VPLANS-EMPTY: ; VPLANS-NEXT: vector.ph: -; VPLANS-NEXT: EMIT vp<[[VF_PER_PART:%.+]]> = VF * Part + ir<0> -; VPLANS-NEXT: EMIT vp<[[LANEMASK_ENTRY:%.+]]> = active lane mask vp<[[VF_PER_PART]]>, vp<[[TC]]> +; VPLANS-NEXT: EMIT vp<[[LANEMASK_ENTRY:%.+]]> = wide active lane mask ir<0>, vp<[[TC]]>, ir<1> +; VPLANS-NEXT: EMIT vp<[[LANEMASK_ENTRY_EXTRACT:%.+]]> = extract-vector-for-part vp<[[LANEMASK_ENTRY]]>, ir<0> ; VPLANS-NEXT: Successor(s): vector loop ; VPLANS-EMPTY: ; VPLANS-NEXT: vector loop: { ; VPLANS-NEXT: vp<[[INDV:%[0-9]+]]> = CANONICAL-IV ; VPLANS-EMPTY: ; VPLANS-NEXT: vector.body: -; VPLANS-NEXT: ACTIVE-LANE-MASK-PHI vp<[[LANEMASK_PHI:%[0-9]+]]> = phi vp<[[LANEMASK_ENTRY]]>, vp<[[LANEMASK_LOOP:%.+]]> +; VPLANS-NEXT: ACTIVE-LANE-MASK-PHI vp<[[LANEMASK_PHI:%[0-9]+]]> = phi vp<[[LANEMASK_ENTRY_EXTRACT]]>, vp<[[LANEMASK_LOOP:%.+]]> ; VPLANS-NEXT: vp<[[STEP:%[0-9]+]]> = SCALAR-STEPS vp<[[INDV]]>, ir<1>, vp<[[VF]]> ; VPLANS-NEXT: CLONE ir<%gep> = getelementptr ir<%ptr>, vp<[[STEP]]> ; VPLANS-NEXT: vp<[[VEC_PTR:%[0-9]+]]> = vector-pointer ir<%gep> ; VPLANS-NEXT: WIDEN store vp<[[VEC_PTR]]>, ir<%val>, vp<[[LANEMASK_PHI]]> ; VPLANS-NEXT: EMIT vp<[[INDV_UPDATE:%.+]]> = add vp<[[INDV]]>, vp<[[VFxUF]]> -; VPLANS-NEXT: EMIT vp<[[INC:%[0-9]+]]> = VF * Part + vp<[[INDV_UPDATE]]>, vp<[[VF]]> -; VPLANS-NEXT: EMIT vp<[[LANEMASK_LOOP]]> = active lane mask vp<[[INC]]>, vp<[[TC]]> +; VPLANS-NEXT: EMIT vp<[[LANEMASK_LOOP_MASK:%.+]]> = wide active lane mask vp<[[INDV_UPDATE]]>, vp<[[TC]]>, ir<1> +; VPLANS-NEXT: EMIT vp<[[LANEMASK_LOOP]]> = extract-vector-for-part vp<[[LANEMASK_LOOP_MASK]]>, ir<0> ; VPLANS-NEXT: EMIT vp<[[NOT:%[0-9]+]]> = not vp<[[LANEMASK_LOOP]]> ; VPLANS-NEXT: EMIT branch-on-cond vp<[[NOT]]> ; VPLANS-NEXT: No successors diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-printing-alias-mask.ll b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-printing-alias-mask.ll index c257a5a14cc27..c40aad62202dd 100644 --- a/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-printing-alias-mask.ll +++ b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-printing-alias-mask.ll @@ -25,12 +25,13 @@ define void @alias_mask(ptr noalias %a, ptr %b, ptr %c, i64 %n) { ; FINAL-NEXT: Successor(s): ir-bb, vector.ph ; FINAL-EMPTY: ; FINAL-NEXT: vector.ph: -; FINAL-NEXT: EMIT vp<%active.lane.mask.entry> = active lane mask ir<0>, ir<%n>, ir<1> +; FINAL-NEXT: EMIT vp<%active.lane.mask.entry> = wide active lane mask ir<0>, ir<%n>, ir<1> +; FINAL-NEXT: EMIT vp<%extract.entry.alm.part> = extract-vector-for-part vp<%active.lane.mask.entry>, ir<0> ; FINAL-NEXT: Successor(s): vector.body ; FINAL-EMPTY: ; FINAL-NEXT: vector.body: ; FINAL-NEXT: EMIT-SCALAR vp<%index> = phi [ ir<0>, vector.ph ], [ vp<%index.next>, vector.body ] -; FINAL-NEXT: ACTIVE-LANE-MASK-PHI vp<[[VP8:%[0-9]+]]> = phi vp<%active.lane.mask.entry>, vp<%active.lane.mask.next> +; FINAL-NEXT: ACTIVE-LANE-MASK-PHI vp<[[VP8:%[0-9]+]]> = phi vp<%extract.entry.alm.part>, vp<%extract.next.alm.part> ; FINAL-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = and vp<[[VP8]]>, vp<[[VP4]]> ; FINAL-NEXT: CLONE ir<%ptr.a> = getelementptr inbounds ir<%a>, vp<%index> ; FINAL-NEXT: WIDEN ir<%ld.a> = load ir<%ptr.a>, vp<[[VP9]]> @@ -40,8 +41,9 @@ define void @alias_mask(ptr noalias %a, ptr %b, ptr %c, i64 %n) { ; FINAL-NEXT: CLONE ir<%ptr.c> = getelementptr inbounds ir<%c>, vp<%index> ; FINAL-NEXT: WIDEN store ir<%ptr.c>, ir<%add>, vp<[[VP9]]> ; FINAL-NEXT: EMIT vp<%index.next> = add vp<%index>, vp<%num.active.lanes> -; FINAL-NEXT: EMIT vp<%active.lane.mask.next> = active lane mask vp<%index.next>, ir<%n>, ir<1> -; FINAL-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = not vp<%active.lane.mask.next> +; FINAL-NEXT: EMIT vp<%active.lane.mask.next> = wide active lane mask vp<%index.next>, ir<%n>, ir<1> +; FINAL-NEXT: EMIT vp<%extract.next.alm.part> = extract-vector-for-part vp<%active.lane.mask.next>, ir<0> +; FINAL-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = not vp<%extract.next.alm.part> ; FINAL-NEXT: EMIT branch-on-cond vp<[[VP10]]> ; FINAL-NEXT: Successor(s): middle.block, vector.body ; FINAL-EMPTY: diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-printing.ll b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-printing.ll index 33b44d49f6be7..f5938862aa006 100644 --- a/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-printing.ll +++ b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-printing.ll @@ -136,15 +136,15 @@ define i32 @print_partial_reduction_predication(ptr %a, ptr %b, i64 %N) "target- ; CHECK-EMPTY: ; CHECK-NEXT: vector.ph: ; CHECK-NEXT: EMIT vp<[[RDX_START:%[0-9]+]]> = reduction-start-vector ir<0>, ir<0>, ir<4> -; CHECK-NEXT: EMIT vp<%index.part.next> = VF * Part + ir<0> -; CHECK-NEXT: EMIT vp<%active.lane.mask.entry> = active lane mask vp<%index.part.next>, ir<%N>, ir<1> +; CHECK-NEXT: EMIT vp<%active.lane.mask.entry> = wide active lane mask ir<0>, ir<%N>, ir<1> +; CHECK-NEXT: EMIT vp<%extract.entry.alm.part> = extract-vector-for-part vp<%active.lane.mask.entry>, ir<0> ; CHECK-NEXT: Successor(s): vector loop ; CHECK-EMPTY: ; CHECK-NEXT: vector loop: { ; CHECK-NEXT: vp<[[CAN_IV:%.+]]> = CANONICAL-IV ; CHECK-EMPTY: ; CHECK-NEXT: vector.body: -; CHECK-NEXT: ACTIVE-LANE-MASK-PHI vp<[[MASK:%[0-9]+]]> = phi vp<%active.lane.mask.entry>, vp<%active.lane.mask.next> +; CHECK-NEXT: ACTIVE-LANE-MASK-PHI vp<[[MASK:%[0-9]+]]> = phi vp<%extract.entry.alm.part>, vp<%extract.next.alm.part> ; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%accum> = phi (add) vp<[[RDX_START]]>, vp<[[REDUCE:%[0-9]+]]> (VF scaled by 1/4) ; CHECK-NEXT: vp<[[STEPS:%[0-9]+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>, vp<[[VF]]> ; CHECK-NEXT: CLONE ir<%gep.a> = getelementptr ir<%a>, vp<[[STEPS]]> @@ -155,9 +155,9 @@ define i32 @print_partial_reduction_predication(ptr %a, ptr %b, i64 %N) "target- ; CHECK-NEXT: WIDEN ir<%load.b> = load vp<[[PTR_B]]>, vp<[[MASK]]> ; CHECK-NEXT: EXPRESSION vp<[[REDUCE]]> = ir<%accum> + partial.reduce.add (mul (ir<%load.b> zext to i32), (ir<%load.a> zext to i32), vp<[[MASK]]>) ; CHECK-NEXT: EMIT vp<%index.next> = add vp<[[CAN_IV]]>, vp<[[VFxUF]]> -; CHECK-NEXT: EMIT vp<[[PART_IDX:%[0-9]+]]> = VF * Part + vp<%index.next>, vp<[[VF]]> -; CHECK-NEXT: EMIT vp<%active.lane.mask.next> = active lane mask vp<[[PART_IDX]]>, ir<%N>, ir<1> -; CHECK-NEXT: EMIT vp<[[NOT_MASK:%[0-9]+]]> = not vp<%active.lane.mask.next> +; CHECK-NEXT: EMIT vp<%active.lane.mask.next> = wide active lane mask vp<%index.next>, ir<%N>, ir<1> +; CHECK-NEXT: EMIT vp<%extract.next.alm.part> = extract-vector-for-part vp<%active.lane.mask.next>, ir<0> +; CHECK-NEXT: EMIT vp<[[NOT_MASK:%[0-9]+]]> = not vp<%extract.next.alm.part> ; CHECK-NEXT: EMIT branch-on-cond vp<[[NOT_MASK]]> ; CHECK-NEXT: No successors ; CHECK-NEXT: } diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/buildvector-first-lane-only.ll b/llvm/test/Transforms/LoopVectorize/VPlan/buildvector-first-lane-only.ll index ded3bc66016fe..b57af70695f6c 100644 --- a/llvm/test/Transforms/LoopVectorize/VPlan/buildvector-first-lane-only.ll +++ b/llvm/test/Transforms/LoopVectorize/VPlan/buildvector-first-lane-only.ll @@ -17,7 +17,7 @@ define i16 @last_active_lane_live_out(i32 %x) { ; CHECK-NEXT: Successor(s): vector.body ; CHECK-EMPTY: ; CHECK-NEXT: vector.body: -; CHECK-NEXT: EMIT vp<%active.lane.mask> = active lane mask ir<0>, ir<2>, ir<1> +; CHECK-NEXT: EMIT vp<%active.lane.mask> = active lane mask ir<0>, ir<2> ; CHECK-NEXT: Successor(s): middle.block ; CHECK-EMPTY: ; CHECK-NEXT: middle.block: diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/early_exit_with_stores_vplan.ll b/llvm/test/Transforms/LoopVectorize/VPlan/early_exit_with_stores_vplan.ll index 822bc1d4f3d93..64c8f2eaf9ecc 100644 --- a/llvm/test/Transforms/LoopVectorize/VPlan/early_exit_with_stores_vplan.ll +++ b/llvm/test/Transforms/LoopVectorize/VPlan/early_exit_with_stores_vplan.ll @@ -28,7 +28,7 @@ define void @loop_contains_store_condition_load_has_single_user(ptr dereferencea ; CHECK-NEXT: WIDEN ir<%ee.val> = load vp<[[VP5]]> ; CHECK-NEXT: WIDEN ir<%ee.cond> = icmp sgt ir<%ee.val>, ir<500> ; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = first-active-lane ir<%ee.cond> -; CHECK-NEXT: EMIT vp<%uncountable.exit.mask> = active lane mask ir<0>, vp<[[VP6]]>, ir<1> +; CHECK-NEXT: EMIT vp<%uncountable.exit.mask> = active lane mask ir<0>, vp<[[VP6]]> ; CHECK-NEXT: CLONE ir<%st.addr> = getelementptr ir<%array>, vp<[[VP4]]> ; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer ir<%st.addr>, ir<1> ; CHECK-NEXT: WIDEN ir<%data> = load vp<[[VP7]]>, vp<%uncountable.exit.mask> @@ -117,7 +117,7 @@ define void @loop_contains_store_after_uncountable_exit(ptr dereferenceable(40) ; CHECK-NEXT: WIDEN ir<%ee.val> = load vp<[[VP5]]> ; CHECK-NEXT: WIDEN ir<%ee.cond> = icmp sgt ir<%ee.val>, ir<500> ; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = first-active-lane ir<%ee.cond> -; CHECK-NEXT: EMIT vp<%uncountable.exit.mask> = active lane mask ir<0>, vp<[[VP6]]>, ir<1> +; CHECK-NEXT: EMIT vp<%uncountable.exit.mask> = active lane mask ir<0>, vp<[[VP6]]> ; CHECK-NEXT: CLONE ir<%st.addr> = getelementptr ir<%array>, vp<[[VP4]]> ; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer ir<%st.addr>, ir<1> ; CHECK-NEXT: WIDEN ir<%data> = load vp<[[VP7]]>, vp<%uncountable.exit.mask> @@ -202,7 +202,7 @@ define i16 @uncountable_exit_with_live_out(ptr dereferenceable(40) noalias %arra ; CHECK-NEXT: WIDEN ir<%ee.val> = load vp<[[VP5]]> ; CHECK-NEXT: WIDEN ir<%ee.cond> = icmp sgt ir<%ee.val>, ir<500> ; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = first-active-lane ir<%ee.cond> -; CHECK-NEXT: EMIT vp<%uncountable.exit.mask> = active lane mask ir<0>, vp<[[VP6]]>, ir<1> +; CHECK-NEXT: EMIT vp<%uncountable.exit.mask> = active lane mask ir<0>, vp<[[VP6]]> ; CHECK-NEXT: CLONE ir<%st.addr> = getelementptr ir<%array>, vp<[[VP4]]> ; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer ir<%st.addr>, ir<1> ; CHECK-NEXT: WIDEN ir<%data> = load vp<[[VP7]]>, vp<%uncountable.exit.mask> diff --git a/llvm/unittests/Transforms/Vectorize/VPlanVerifierTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanVerifierTest.cpp index ebf7a0942ae2c..eda6157fde93f 100644 --- a/llvm/unittests/Transforms/Vectorize/VPlanVerifierTest.cpp +++ b/llvm/unittests/Transforms/Vectorize/VPlanVerifierTest.cpp @@ -514,4 +514,46 @@ TEST_F(VPIRVerifierTest, BranchOnTwoCondsLatchHeaderVerification) { #endif } +TEST_F(VPIRVerifierTest, testVerifyLastActiveWideActiveLaneMask) { + const char *ModuleString = + "define i32 @f(ptr %A, i64 %N) {\n" + "entry:\n" + " br label %loop\n" + "loop:\n" + " %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]\n" + " %gep = getelementptr i32, ptr %A, i64 %iv\n" + " %ld = load i32, ptr %gep \n" + " %iv.next = add i64 %iv, 1\n" + " %exitcond = icmp eq i64 %iv.next, %N\n" + " br i1 %exitcond, label %exit, label %loop\n" + "exit:\n" + " ret i32 %ld\n" + "}\n"; + + Module &M = parseModule(ModuleString); + + Function *F = M.getFunction("f"); + BasicBlock *LoopHeader = F->getEntryBlock().getSingleSuccessor(); + auto Plan = buildVPlan(LoopHeader); + + VPValue *Start = Plan->getConstantInt(64, 0); + VPValue *TC = Plan->getTripCount(); + VPValue *One = Plan->getConstantInt(64, 1); + Type *MaskTy = FixedVectorType::get(Type::getInt1Ty(*Ctx), 4); + Type *IndexTy = Plan->getDataLayout().getIndexType(Plan->getContext(), 0); + + auto *WideALM = + new VPInstruction(VPInstruction::WideActiveLaneMask, {Start, TC, One}, {}, + {}, DebugLoc(), "", MaskTy); + auto *Preheader = Plan->getVectorPreheader(); + WideALM->insertBefore(*Preheader, Preheader->end()); + + auto *LastActiveLane = + new VPInstruction(VPInstruction::LastActiveLane, {WideALM}, {}, {}, + DebugLoc(), "", IndexTy); + LastActiveLane->insertBefore(Plan->getMiddleBlock()->getTerminator()); + + EXPECT_TRUE(verifyVPlanIsValid(*Plan)); +} + } // namespace