234 changes: 128 additions & 106 deletions llvm/lib/Analysis/IVDescriptors.cpp
View file

Large diffs are not rendered by default.

26 changes: 13 additions & 13 deletions llvm/lib/CodeGen/ExpandReductions.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,22 +57,22 @@ unsigned getOpcode(Intrinsic::ID ID) {
}
}

RecurrenceDescriptor::MinMaxRecurrenceKind getMRK(Intrinsic::ID ID) {
RecurKind getRK(Intrinsic::ID ID) {
switch (ID) {
case Intrinsic::vector_reduce_smax:
return RecurrenceDescriptor::MRK_SIntMax;
return RecurKind::SMax;
case Intrinsic::vector_reduce_smin:
return RecurrenceDescriptor::MRK_SIntMin;
return RecurKind::SMin;
case Intrinsic::vector_reduce_umax:
return RecurrenceDescriptor::MRK_UIntMax;
return RecurKind::UMax;
case Intrinsic::vector_reduce_umin:
return RecurrenceDescriptor::MRK_UIntMin;
return RecurKind::UMin;
case Intrinsic::vector_reduce_fmax:
return RecurrenceDescriptor::MRK_FloatMax;
return RecurKind::FMax;
case Intrinsic::vector_reduce_fmin:
return RecurrenceDescriptor::MRK_FloatMin;
return RecurKind::FMin;
default:
return RecurrenceDescriptor::MRK_Invalid;
return RecurKind::None;
}
}

Expand Down Expand Up @@ -108,7 +108,7 @@ bool expandReductions(Function &F, const TargetTransformInfo *TTI) {
FastMathFlags FMF =
isa<FPMathOperator>(II) ? II->getFastMathFlags() : FastMathFlags{};
Intrinsic::ID ID = II->getIntrinsicID();
RecurrenceDescriptor::MinMaxRecurrenceKind MRK = getMRK(ID);
RecurKind RK = getRK(ID);

Value *Rdx = nullptr;
IRBuilder<> Builder(II);
Expand All @@ -123,13 +123,13 @@ bool expandReductions(Function &F, const TargetTransformInfo *TTI) {
Value *Acc = II->getArgOperand(0);
Value *Vec = II->getArgOperand(1);
if (!FMF.allowReassoc())
Rdx = getOrderedReduction(Builder, Acc, Vec, getOpcode(ID), MRK);
Rdx = getOrderedReduction(Builder, Acc, Vec, getOpcode(ID), RK);
else {
if (!isPowerOf2_32(
cast<FixedVectorType>(Vec->getType())->getNumElements()))
continue;

Rdx = getShuffleReduction(Builder, Vec, getOpcode(ID), MRK);
Rdx = getShuffleReduction(Builder, Vec, getOpcode(ID), RK);
Rdx = Builder.CreateBinOp((Instruction::BinaryOps)getOpcode(ID),
Acc, Rdx, "bin.rdx");
}
Expand All @@ -149,7 +149,7 @@ bool expandReductions(Function &F, const TargetTransformInfo *TTI) {
cast<FixedVectorType>(Vec->getType())->getNumElements()))
continue;

Rdx = getShuffleReduction(Builder, Vec, getOpcode(ID), MRK);
Rdx = getShuffleReduction(Builder, Vec, getOpcode(ID), RK);
break;
}
case Intrinsic::vector_reduce_fmax:
Expand All @@ -163,7 +163,7 @@ bool expandReductions(Function &F, const TargetTransformInfo *TTI) {
!FMF.isFast())
continue;

Rdx = getShuffleReduction(Builder, Vec, getOpcode(ID), MRK);
Rdx = getShuffleReduction(Builder, Vec, getOpcode(ID), RK);
break;
}
}
Expand Down
84 changes: 36 additions & 48 deletions llvm/lib/Transforms/Utils/LoopUtils.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -868,29 +868,28 @@ bool llvm::hasIterationCountInvariantInParent(Loop *InnerLoop,
return true;
}

Value *llvm::createMinMaxOp(IRBuilderBase &Builder,
RecurrenceDescriptor::MinMaxRecurrenceKind RK,
Value *Left, Value *Right) {
Value *llvm::createMinMaxOp(IRBuilderBase &Builder, RecurKind RK, Value *Left,
Value *Right) {
CmpInst::Predicate P = CmpInst::ICMP_NE;
switch (RK) {
default:
llvm_unreachable("Unknown min/max recurrence kind");
case RecurrenceDescriptor::MRK_UIntMin:
case RecurKind::UMin:
P = CmpInst::ICMP_ULT;
break;
case RecurrenceDescriptor::MRK_UIntMax:
case RecurKind::UMax:
P = CmpInst::ICMP_UGT;
break;
case RecurrenceDescriptor::MRK_SIntMin:
case RecurKind::SMin:
P = CmpInst::ICMP_SLT;
break;
case RecurrenceDescriptor::MRK_SIntMax:
case RecurKind::SMax:
P = CmpInst::ICMP_SGT;
break;
case RecurrenceDescriptor::MRK_FloatMin:
case RecurKind::FMin:
P = CmpInst::FCMP_OLT;
break;
case RecurrenceDescriptor::MRK_FloatMax:
case RecurKind::FMax:
P = CmpInst::FCMP_OGT;
break;
}
Expand All @@ -907,11 +906,9 @@ Value *llvm::createMinMaxOp(IRBuilderBase &Builder,
}

// Helper to generate an ordered reduction.
Value *
llvm::getOrderedReduction(IRBuilderBase &Builder, Value *Acc, Value *Src,
unsigned Op,
RecurrenceDescriptor::MinMaxRecurrenceKind MinMaxKind,
ArrayRef<Value *> RedOps) {
Value *llvm::getOrderedReduction(IRBuilderBase &Builder, Value *Acc, Value *Src,
unsigned Op, RecurKind RdxKind,
ArrayRef<Value *> RedOps) {
unsigned VF = cast<FixedVectorType>(Src->getType())->getNumElements();

// Extract and apply reduction ops in ascending order:
Expand All @@ -925,9 +922,9 @@ llvm::getOrderedReduction(IRBuilderBase &Builder, Value *Acc, Value *Src,
Result = Builder.CreateBinOp((Instruction::BinaryOps)Op, Result, Ext,
"bin.rdx");
} else {
assert(MinMaxKind != RecurrenceDescriptor::MRK_Invalid &&
assert(RecurrenceDescriptor::isMinMaxRecurrenceKind(RdxKind) &&
"Invalid min/max");
Result = createMinMaxOp(Builder, MinMaxKind, Result, Ext);
Result = createMinMaxOp(Builder, RdxKind, Result, Ext);
}

if (!RedOps.empty())
Expand All @@ -938,10 +935,9 @@ llvm::getOrderedReduction(IRBuilderBase &Builder, Value *Acc, Value *Src,
}

// Helper to generate a log2 shuffle reduction.
Value *
llvm::getShuffleReduction(IRBuilderBase &Builder, Value *Src, unsigned Op,
RecurrenceDescriptor::MinMaxRecurrenceKind MinMaxKind,
ArrayRef<Value *> RedOps) {
Value *llvm::getShuffleReduction(IRBuilderBase &Builder, Value *Src,
unsigned Op, RecurKind RdxKind,
ArrayRef<Value *> RedOps) {
unsigned VF = cast<FixedVectorType>(Src->getType())->getNumElements();
// VF is a power of 2 so we can emit the reduction using log2(VF) shuffles
// and vector ops, reducing the set of values being computed by half each
Expand All @@ -965,9 +961,9 @@ llvm::getShuffleReduction(IRBuilderBase &Builder, Value *Src, unsigned Op,
TmpVec = Builder.CreateBinOp((Instruction::BinaryOps)Op, TmpVec, Shuf,
"bin.rdx");
} else {
assert(MinMaxKind != RecurrenceDescriptor::MRK_Invalid &&
assert(RecurrenceDescriptor::isMinMaxRecurrenceKind(RdxKind) &&
"Invalid min/max");
TmpVec = createMinMaxOp(Builder, MinMaxKind, TmpVec, Shuf);
TmpVec = createMinMaxOp(Builder, RdxKind, TmpVec, Shuf);
}
if (!RedOps.empty())
propagateIRFlags(TmpVec, RedOps);
Expand All @@ -981,16 +977,14 @@ llvm::getShuffleReduction(IRBuilderBase &Builder, Value *Src, unsigned Op,
return Builder.CreateExtractElement(TmpVec, Builder.getInt32(0));
}

/// Create a simple vector reduction specified by an opcode and some
/// flags (if generating min/max reductions).
Value *llvm::createSimpleTargetReduction(
IRBuilderBase &Builder, const TargetTransformInfo *TTI, unsigned Opcode,
Value *Src, RecurrenceDescriptor::MinMaxRecurrenceKind MinMaxKind,
ArrayRef<Value *> RedOps) {
Value *llvm::createSimpleTargetReduction(IRBuilderBase &Builder,
const TargetTransformInfo *TTI,
unsigned Opcode, Value *Src,
RecurKind RdxKind,
ArrayRef<Value *> RedOps) {
auto *SrcVTy = cast<VectorType>(Src->getType());

std::function<Value *()> BuildFunc;
using RD = RecurrenceDescriptor;
switch (Opcode) {
case Instruction::Add:
BuildFunc = [&]() { return Builder.CreateAddReduce(Src); };
Expand Down Expand Up @@ -1022,27 +1016,27 @@ Value *llvm::createSimpleTargetReduction(
};
break;
case Instruction::ICmp:
switch (MinMaxKind) {
case RD::MRK_SIntMax:
switch (RdxKind) {
case RecurKind::SMax:
BuildFunc = [&]() { return Builder.CreateIntMaxReduce(Src, true); };
break;
case RD::MRK_SIntMin:
case RecurKind::SMin:
BuildFunc = [&]() { return Builder.CreateIntMinReduce(Src, true); };
break;
case RD::MRK_UIntMax:
case RecurKind::UMax:
BuildFunc = [&]() { return Builder.CreateIntMaxReduce(Src, false); };
break;
case RD::MRK_UIntMin:
case RecurKind::UMin:
BuildFunc = [&]() { return Builder.CreateIntMinReduce(Src, false); };
break;
default:
llvm_unreachable("Unexpected min/max reduction type");
}
break;
case Instruction::FCmp:
assert((MinMaxKind == RD::MRK_FloatMax || MinMaxKind == RD::MRK_FloatMin) &&
assert((RdxKind == RecurKind::FMax || RdxKind == RecurKind::FMin) &&
"Unexpected min/max reduction type");
if (MinMaxKind == RD::MRK_FloatMax)
if (RdxKind == RecurKind::FMax)
BuildFunc = [&]() { return Builder.CreateFPMaxReduce(Src); };
else
BuildFunc = [&]() { return Builder.CreateFPMinReduce(Src); };
Expand All @@ -1051,32 +1045,26 @@ Value *llvm::createSimpleTargetReduction(
llvm_unreachable("Unhandled opcode");
}
TargetTransformInfo::ReductionFlags RdxFlags;
RdxFlags.IsMaxOp = MinMaxKind == RD::MRK_SIntMax ||
MinMaxKind == RD::MRK_UIntMax ||
MinMaxKind == RD::MRK_FloatMax;
RdxFlags.IsSigned =
MinMaxKind == RD::MRK_SIntMax || MinMaxKind == RD::MRK_SIntMin;
RdxFlags.IsMaxOp = RdxKind == RecurKind::SMax ||
RdxKind == RecurKind::UMax ||
RdxKind == RecurKind::FMax;
RdxFlags.IsSigned = RdxKind == RecurKind::SMax || RdxKind == RecurKind::SMin;
if (ForceReductionIntrinsic ||
TTI->useReductionIntrinsic(Opcode, Src->getType(), RdxFlags))
return BuildFunc();
return getShuffleReduction(Builder, Src, Opcode, MinMaxKind, RedOps);
return getShuffleReduction(Builder, Src, Opcode, RdxKind, RedOps);
}

/// Create a vector reduction using a given recurrence descriptor.
Value *llvm::createTargetReduction(IRBuilderBase &B,
const TargetTransformInfo *TTI,
RecurrenceDescriptor &Desc, Value *Src) {
// TODO: Support in-order reductions based on the recurrence descriptor.
using RD = RecurrenceDescriptor;

// All ops in the reduction inherit fast-math-flags from the recurrence
// descriptor.
IRBuilderBase::FastMathFlagGuard FMFGuard(B);
B.setFastMathFlags(Desc.getFastMathFlags());

RD::MinMaxRecurrenceKind MMKind = Desc.getMinMaxRecurrenceKind();
return createSimpleTargetReduction(B, TTI, Desc.getRecurrenceBinOp(), Src,
MMKind);
Desc.getRecurrenceKind());
}

void llvm::propagateIRFlags(Value *I, ArrayRef<Value *> VL, Value *OpValue) {
Expand Down
41 changes: 15 additions & 26 deletions llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4152,11 +4152,9 @@ void InnerLoopVectorizer::fixReduction(PHINode *Phi) {
"Unable to find the reduction variable");
RecurrenceDescriptor RdxDesc = Legal->getReductionVars()[Phi];

RecurrenceDescriptor::RecurrenceKind RK = RdxDesc.getRecurrenceKind();
RecurKind RK = RdxDesc.getRecurrenceKind();
TrackingVH<Value> ReductionStartValue = RdxDesc.getRecurrenceStartValue();
Instruction *LoopExitInst = RdxDesc.getLoopExitInstr();
RecurrenceDescriptor::MinMaxRecurrenceKind MinMaxKind =
RdxDesc.getMinMaxRecurrenceKind();
setDebugLocFromInst(Builder, ReductionStartValue);
bool IsInLoopReductionPhi = Cost->isInLoopReduction(Phi);

Expand All @@ -4173,8 +4171,7 @@ void InnerLoopVectorizer::fixReduction(PHINode *Phi) {
// one for multiplication, -1 for And.
Value *Identity;
Value *VectorStart;
if (RK == RecurrenceDescriptor::RK_IntegerMinMax ||
RK == RecurrenceDescriptor::RK_FloatMinMax) {
if (RecurrenceDescriptor::isMinMaxRecurrenceKind(RK)) {
// MinMax reduction have the start value as their identify.
if (VF.isScalar() || IsInLoopReductionPhi) {
VectorStart = Identity = ReductionStartValue;
Expand All @@ -4185,7 +4182,7 @@ void InnerLoopVectorizer::fixReduction(PHINode *Phi) {
} else {
// Handle other reduction kinds:
Constant *Iden = RecurrenceDescriptor::getRecurrenceIdentity(
RK, MinMaxKind, VecTy->getScalarType());
RK, VecTy->getScalarType());
if (VF.isScalar() || IsInLoopReductionPhi) {
Identity = Iden;
// This vector is the Identity vector where the first element is the
Expand Down Expand Up @@ -4318,8 +4315,7 @@ void InnerLoopVectorizer::fixReduction(PHINode *Phi) {
ReducedPartRdx, "bin.rdx"),
RdxDesc.getFastMathFlags());
else
ReducedPartRdx = createMinMaxOp(Builder, MinMaxKind, ReducedPartRdx,
RdxPart);
ReducedPartRdx = createMinMaxOp(Builder, RK, ReducedPartRdx, RdxPart);
}

// Create the reduction after the loop. Note that inloop reductions create the
Expand Down Expand Up @@ -4372,9 +4368,8 @@ void InnerLoopVectorizer::fixReduction(PHINode *Phi) {

void InnerLoopVectorizer::clearReductionWrapFlags(
RecurrenceDescriptor &RdxDesc) {
RecurrenceDescriptor::RecurrenceKind RK = RdxDesc.getRecurrenceKind();
if (RK != RecurrenceDescriptor::RK_IntegerAdd &&
RK != RecurrenceDescriptor::RK_IntegerMult)
RecurKind RK = RdxDesc.getRecurrenceKind();
if (RK != RecurKind::Add && RK != RecurKind::Mul)
return;

Instruction *LoopExitInstr = RdxDesc.getLoopExitInstr();
Expand Down Expand Up @@ -8395,19 +8390,16 @@ VPlanPtr LoopVectorizationPlanner::buildVPlanWithVPRecipes(
}
for (auto &Reduction : CM.getInLoopReductionChains()) {
PHINode *Phi = Reduction.first;
RecurrenceDescriptor::RecurrenceKind Kind =
Legal->getReductionVars()[Phi].getRecurrenceKind();
RecurKind Kind = Legal->getReductionVars()[Phi].getRecurrenceKind();
const SmallVector<Instruction *, 4> &ReductionOperations = Reduction.second;

RecipeBuilder.recordRecipeOf(Phi);
for (auto &R : ReductionOperations) {
RecipeBuilder.recordRecipeOf(R);
// For min/max reducitons, where we have a pair of icmp/select, we also
// need to record the ICmp recipe, so it can be removed later.
if (Kind == RecurrenceDescriptor::RK_IntegerMinMax ||
Kind == RecurrenceDescriptor::RK_FloatMinMax) {
if (RecurrenceDescriptor::isMinMaxRecurrenceKind(Kind))
RecipeBuilder.recordRecipeOf(cast<Instruction>(R->getOperand(0)));
}
}
}

Expand Down Expand Up @@ -8621,12 +8613,11 @@ void LoopVectorizationPlanner::adjustRecipesForInLoopReductions(
Instruction *Chain = Phi;
for (Instruction *R : ReductionOperations) {
VPRecipeBase *WidenRecipe = RecipeBuilder.getRecipe(R);
RecurrenceDescriptor::RecurrenceKind Kind = RdxDesc.getRecurrenceKind();
RecurKind Kind = RdxDesc.getRecurrenceKind();

VPValue *ChainOp = Plan->getVPValue(Chain);
unsigned FirstOpId;
if (Kind == RecurrenceDescriptor::RK_IntegerMinMax ||
Kind == RecurrenceDescriptor::RK_FloatMinMax) {
if (RecurrenceDescriptor::isMinMaxRecurrenceKind(Kind)) {
assert(isa<VPWidenSelectRecipe>(WidenRecipe) &&
"Expected to replace a VPWidenSelectSC");
FirstOpId = 1;
Expand All @@ -8651,8 +8642,7 @@ void LoopVectorizationPlanner::adjustRecipesForInLoopReductions(
WidenRecipe->getVPValue()->replaceAllUsesWith(RedRecipe);
WidenRecipe->eraseFromParent();

if (Kind == RecurrenceDescriptor::RK_IntegerMinMax ||
Kind == RecurrenceDescriptor::RK_FloatMinMax) {
if (RecurrenceDescriptor::isMinMaxRecurrenceKind(Kind)) {
VPRecipeBase *CompareRecipe =
RecipeBuilder.getRecipe(cast<Instruction>(R->getOperand(0)));
assert(isa<VPWidenRecipe>(CompareRecipe) &&
Expand Down Expand Up @@ -8769,13 +8759,13 @@ void VPInterleaveRecipe::execute(VPTransformState &State) {
void VPReductionRecipe::execute(VPTransformState &State) {
assert(!State.Instance && "Reduction being replicated.");
for (unsigned Part = 0; Part < State.UF; ++Part) {
RecurrenceDescriptor::RecurrenceKind Kind = RdxDesc->getRecurrenceKind();
RecurKind Kind = RdxDesc->getRecurrenceKind();
Value *NewVecOp = State.get(getVecOp(), Part);
if (VPValue *Cond = getCondOp()) {
Value *NewCond = State.get(Cond, Part);
VectorType *VecTy = cast<VectorType>(NewVecOp->getType());
Constant *Iden = RecurrenceDescriptor::getRecurrenceIdentity(
Kind, RdxDesc->getMinMaxRecurrenceKind(), VecTy->getElementType());
Kind, VecTy->getElementType());
Constant *IdenVec =
ConstantVector::getSplat(VecTy->getElementCount(), Iden);
Value *Select = State.Builder.CreateSelect(NewCond, NewVecOp, IdenVec);
Expand All @@ -8785,10 +8775,9 @@ void VPReductionRecipe::execute(VPTransformState &State) {
createTargetReduction(State.Builder, TTI, *RdxDesc, NewVecOp);
Value *PrevInChain = State.get(getChainOp(), Part);
Value *NextInChain;
if (Kind == RecurrenceDescriptor::RK_IntegerMinMax ||
Kind == RecurrenceDescriptor::RK_FloatMinMax) {
if (RecurrenceDescriptor::isMinMaxRecurrenceKind(Kind)) {
NextInChain =
createMinMaxOp(State.Builder, RdxDesc->getMinMaxRecurrenceKind(),
createMinMaxOp(State.Builder, RdxDesc->getRecurrenceKind(),
NewRed, PrevInChain);
} else {
NextInChain = State.Builder.CreateBinOp(
Expand Down
224 changes: 116 additions & 108 deletions llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
View file

Large diffs are not rendered by default.