From e64030f5157a0aff8c4d56ad241c90147f4020a9 Mon Sep 17 00:00:00 2001 From: Mel Chen Date: Wed, 15 Oct 2025 03:13:38 -0700 Subject: [PATCH] [LV] Explicitly disable in-loop reductions for AnyOf and FindIV --- llvm/lib/Analysis/IVDescriptors.cpp | 12 +++++++----- llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 7 ++++++- llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp | 7 +++---- 3 files changed, 16 insertions(+), 10 deletions(-) diff --git a/llvm/lib/Analysis/IVDescriptors.cpp b/llvm/lib/Analysis/IVDescriptors.cpp index 9f8ac6e8e2e0b..641850b46bbd8 100644 --- a/llvm/lib/Analysis/IVDescriptors.cpp +++ b/llvm/lib/Analysis/IVDescriptors.cpp @@ -1220,11 +1220,6 @@ unsigned RecurrenceDescriptor::getOpcode(RecurKind Kind) { return Instruction::Add; case RecurKind::Mul: return Instruction::Mul; - case RecurKind::AnyOf: - case RecurKind::FindFirstIVSMin: - case RecurKind::FindFirstIVUMin: - case RecurKind::FindLastIVSMax: - case RecurKind::FindLastIVUMax: case RecurKind::Or: return Instruction::Or; case RecurKind::And: @@ -1248,6 +1243,13 @@ unsigned RecurrenceDescriptor::getOpcode(RecurKind Kind) { case RecurKind::FMaximumNum: case RecurKind::FMinimumNum: return Instruction::FCmp; + case RecurKind::AnyOf: + case RecurKind::FindFirstIVSMin: + case RecurKind::FindFirstIVUMin: + case RecurKind::FindLastIVSMax: + case RecurKind::FindLastIVUMax: + // TODO: Set AnyOf and FindIV to Instruction::Select once in-loop reductions + // are supported. default: llvm_unreachable("Unknown recurrence operation"); } diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index b96d29e635465..77916a0ea744d 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -6515,9 +6515,14 @@ void LoopVectorizationCostModel::collectInLoopReductions() { if (RdxDesc.getRecurrenceType() != Phi->getType()) continue; + // In-loop AnyOf and FindIV reductions are not yet supported. + RecurKind Kind = RdxDesc.getRecurrenceKind(); + if (RecurrenceDescriptor::isAnyOfRecurrenceKind(Kind) || + RecurrenceDescriptor::isFindIVRecurrenceKind(Kind)) + continue; + // If the target would prefer this reduction to happen "in-loop", then we // want to record it as such. - RecurKind Kind = RdxDesc.getRecurrenceKind(); if (!PreferInLoopReductions && !useOrderedReductions(RdxDesc) && !TTI.preferInLoopReduction(Kind, Phi->getType())) continue; diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp index 2368d18b0373c..95d0b899b9085 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp @@ -784,10 +784,9 @@ Value *VPInstruction::generate(VPTransformState &State) { auto *OrigPhi = cast(PhiR->getUnderlyingValue()); Value *ReducedPartRdx = State.get(getOperand(2)); for (unsigned Idx = 3; Idx < getNumOperands(); ++Idx) - ReducedPartRdx = Builder.CreateBinOp( - (Instruction::BinaryOps)RecurrenceDescriptor::getOpcode( - RecurKind::AnyOf), - State.get(getOperand(Idx)), ReducedPartRdx, "bin.rdx"); + ReducedPartRdx = + Builder.CreateBinOp(Instruction::Or, State.get(getOperand(Idx)), + ReducedPartRdx, "bin.rdx"); return createAnyOfReduction(Builder, ReducedPartRdx, State.get(getOperand(1), VPLane(0)), OrigPhi); }