diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index 651a201e4ff947..c807ec10c99ec5 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -410,7 +410,8 @@ class InnerLoopVectorizer { void widenInstruction(Instruction &I); /// Widen a single call instruction within the innermost loop. - void widenCallInstruction(CallInst &I); + void widenCallInstruction(CallInst &I, VPUser &ArgOperands, + VPTransformState &State); /// Fix the vectorized code, taking care of header phi's, live-outs, and more. void fixVectorizedLoop(); @@ -4359,7 +4360,8 @@ void InnerLoopVectorizer::widenInstruction(Instruction &I) { } // end of switch. } -void InnerLoopVectorizer::widenCallInstruction(CallInst &I) { +void InnerLoopVectorizer::widenCallInstruction(CallInst &I, VPUser &ArgOperands, + VPTransformState &State) { // Ignore dbg intrinsics. // TODO: Debug intrinsics should be skipped/handled during VPlan construction // rather than dropping them here. @@ -4388,12 +4390,14 @@ void InnerLoopVectorizer::widenCallInstruction(CallInst &I) { for (unsigned Part = 0; Part < UF; ++Part) { SmallVector Args; - for (unsigned i = 0, ie = CI->getNumArgOperands(); i != ie; ++i) { - Value *Arg = CI->getArgOperand(i); + for (auto &I : enumerate(ArgOperands.operands())) { // Some intrinsics have a scalar argument - don't replace it with a // vector. - if (!UseVectorIntrinsic || !hasVectorInstrinsicScalarOpd(ID, i)) - Arg = getOrCreateVectorValue(CI->getArgOperand(i), Part); + Value *Arg; + if (!UseVectorIntrinsic || !hasVectorInstrinsicScalarOpd(ID, I.index())) + Arg = State.get(I.value(), Part); + else + Arg = State.get(I.value(), {0, 0}); Args.push_back(Arg); } @@ -6887,8 +6891,8 @@ VPBlendRecipe *VPRecipeBuilder::tryToBlend(Instruction *I, VPlanPtr &Plan) { return new VPBlendRecipe(Phi, Masks); } -VPWidenCallRecipe *VPRecipeBuilder::tryToWidenCall(Instruction *I, - VFRange &Range) { +VPWidenCallRecipe * +VPRecipeBuilder::tryToWidenCall(Instruction *I, VFRange &Range, VPlan &Plan) { bool IsPredicated = LoopVectorizationPlanner::getDecisionAndClampRange( [&](unsigned VF) { return CM.isScalarWithPredication(I, VF); }, Range); @@ -6919,7 +6923,11 @@ VPWidenCallRecipe *VPRecipeBuilder::tryToWidenCall(Instruction *I, return nullptr; // Success: widen this call. - return new VPWidenCallRecipe(*CI); + auto VPValues = map_range(CI->arg_operands(), [&Plan](Value *Op) { + return Plan.getOrAddVPValue(Op); + }); + + return new VPWidenCallRecipe(*CI, VPValues); } VPWidenRecipe *VPRecipeBuilder::tryToWiden(Instruction *I, VFRange &Range) { @@ -7071,7 +7079,7 @@ bool VPRecipeBuilder::tryToCreateRecipe(Instruction *Instr, VFRange &Range, // First, check for specific widening recipes that deal with calls, memory // operations, inductions and Phi nodes. - if ((Recipe = tryToWidenCall(Instr, Range)) || + if ((Recipe = tryToWidenCall(Instr, Range, *Plan)) || (Recipe = tryToWidenMemory(Instr, Range, Plan)) || (Recipe = tryToOptimizeInduction(Instr, Range)) || (Recipe = tryToBlend(Instr, Plan)) || @@ -7390,7 +7398,7 @@ void VPInterleaveRecipe::print(raw_ostream &O, const Twine &Indent, } void VPWidenCallRecipe::execute(VPTransformState &State) { - State.ILV->widenCallInstruction(Ingredient); + State.ILV->widenCallInstruction(Ingredient, User, State); } void VPWidenRecipe::execute(VPTransformState &State) { diff --git a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h index d5435c23e65c23..805f11d1ebb1d6 100644 --- a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h +++ b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h @@ -110,7 +110,8 @@ class VPRecipeBuilder { /// Handle call instruction. If \p I is a call that can be widened for \p /// Range.Start, return a new VPWidenCallRecipe. Range.End may be decreased to /// ensure same decision from \p Range.Start to \p Range.End. - VPWidenCallRecipe *tryToWidenCall(Instruction *I, VFRange &Range); + VPWidenCallRecipe *tryToWidenCall(Instruction *I, VFRange &Range, + VPlan &Plan); /// Check if \p I can be widened within the given VF \p Range. If \p I can be /// widened for \p Range.Start, build a new VPWidenRecipe and return it. diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h index 7e1f7473591889..ed59f6e61b6740 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -790,8 +790,13 @@ class VPWidenCallRecipe : public VPRecipeBase { /// Hold the call to be widened. CallInst &Ingredient; + /// Hold VPValues for the arguments of the call. + VPUser User; + public: - VPWidenCallRecipe(CallInst &I) : VPRecipeBase(VPWidenCallSC), Ingredient(I) {} + template + VPWidenCallRecipe(CallInst &I, iterator_range CallArguments) + : VPRecipeBase(VPWidenCallSC), Ingredient(I), User(CallArguments) {} ~VPWidenCallRecipe() override = default; diff --git a/llvm/lib/Transforms/Vectorize/VPlanValue.h b/llvm/lib/Transforms/Vectorize/VPlanValue.h index abaf4dc43015cf..7afbb97ca548ee 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanValue.h +++ b/llvm/lib/Transforms/Vectorize/VPlanValue.h @@ -22,6 +22,7 @@ #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/iterator_range.h" #include "llvm/IR/Value.h" #include "llvm/Support/Debug.h" #include "llvm/Support/raw_ostream.h" @@ -147,6 +148,12 @@ class VPUser : public VPValue { VPUser(ArrayRef Operands) : VPUser(VPValue::VPUserSC, Operands) {} VPUser(std::initializer_list Operands) : VPUser(ArrayRef(Operands)) {} + template + VPUser(iterator_range Operands) : VPValue(VPValue::VPUserSC) { + for (VPValue *Operand : Operands) + addOperand(Operand); + } + VPUser(const VPUser &) = delete; VPUser &operator=(const VPUser &) = delete;