diff --git a/llvm/include/llvm/Transforms/Vectorize/SLPVectorizer.h b/llvm/include/llvm/Transforms/Vectorize/SLPVectorizer.h index 8cd9781831a45..fa35ff8d5afb4 100644 --- a/llvm/include/llvm/Transforms/Vectorize/SLPVectorizer.h +++ b/llvm/include/llvm/Transforms/Vectorize/SLPVectorizer.h @@ -144,6 +144,10 @@ struct SLPVectorizerPass : public PassInfoMixin { bool vectorizeInsertElementInst(InsertElementInst *IEI, BasicBlock *BB, slpvectorizer::BoUpSLP &R); + /// Tries to vectorize \p CmpInts. \Returns true on success. + bool vectorizeCmpInsts(ArrayRef CmpInsts, BasicBlock *BB, + slpvectorizer::BoUpSLP &R); + /// Tries to vectorize constructs started from CmpInst, InsertValueInst or /// InsertElementInst instructions. bool vectorizeSimpleInstructions(InstSetVector &Instructions, BasicBlock *BB, diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp index a186ce9eddd3a..0823086e47ade 100644 --- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp +++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp @@ -14392,18 +14392,74 @@ static bool compareCmp(Value *V, Value *V2, TargetLibraryInfo &TLI, return IsCompatibility; } +bool SLPVectorizerPass::vectorizeCmpInsts(ArrayRef CmpInsts, + BasicBlock *BB, BoUpSLP &R) { + bool Changed = false; + // Try to find reductions first. + for (Instruction *I : CmpInsts) { + if (R.isDeleted(I)) + continue; + for (Value *Op : I->operands()) + if (auto *RootOp = dyn_cast(Op)) + Changed |= vectorizeRootInstruction(nullptr, RootOp, BB, R, TTI); + } + // Try to vectorize operands as vector bundles. + for (Instruction *I : CmpInsts) { + if (R.isDeleted(I)) + continue; + Changed |= tryToVectorize(I, R); + } + // Try to vectorize list of compares. + // Sort by type, compare predicate, etc. + auto CompareSorter = [&](Value *V, Value *V2) { + return compareCmp(V, V2, *TLI, + [&R](Instruction *I) { return R.isDeleted(I); }); + }; + + auto AreCompatibleCompares = [&](Value *V1, Value *V2) { + if (V1 == V2) + return true; + return compareCmp(V1, V2, *TLI, + [&R](Instruction *I) { return R.isDeleted(I); }); + }; + + SmallVector Vals(CmpInsts.begin(), CmpInsts.end()); + Changed |= tryToVectorizeSequence( + Vals, CompareSorter, AreCompatibleCompares, + [this, &R](ArrayRef Candidates, bool LimitForRegisterSize) { + // Exclude possible reductions from other blocks. + bool ArePossiblyReducedInOtherBlock = any_of(Candidates, [](Value *V) { + return any_of(V->users(), [V](User *U) { + auto *Select = dyn_cast(U); + return Select && + Select->getParent() != cast(V)->getParent(); + }); + }); + if (ArePossiblyReducedInOtherBlock) + return false; + return tryToVectorizeList(Candidates, R, LimitForRegisterSize); + }, + /*LimitForRegisterSize=*/true, R); + return Changed; +} + bool SLPVectorizerPass::vectorizeSimpleInstructions(InstSetVector &Instructions, BasicBlock *BB, BoUpSLP &R, bool AtTerminator) { + assert(all_of(Instructions, + [](auto *I) { + return isa(I); + }) && + "This function only accepts Cmp and Insert instructions"); bool OpsChanged = false; - SmallVector PostponedCmps; + SmallVector PostponedCmps; SmallVector PostponedInsts; // pass1 - try to vectorize reductions only for (auto *I : reverse(Instructions)) { if (R.isDeleted(I)) continue; if (isa(I)) { - PostponedCmps.push_back(I); + PostponedCmps.push_back(cast(I)); continue; } OpsChanged |= vectorizeHorReduction(nullptr, I, BB, R, TTI, PostponedInsts); @@ -14422,52 +14478,7 @@ bool SLPVectorizerPass::vectorizeSimpleInstructions(InstSetVector &Instructions, OpsChanged |= tryToVectorize(PostponedInsts, R); if (AtTerminator) { - // Try to find reductions first. - for (Instruction *I : PostponedCmps) { - if (R.isDeleted(I)) - continue; - for (Value *Op : I->operands()) - if (auto *RootOp = dyn_cast(Op)) - OpsChanged |= vectorizeRootInstruction(nullptr, RootOp, BB, R, TTI); - } - // Try to vectorize operands as vector bundles. - for (Instruction *I : PostponedCmps) { - if (R.isDeleted(I)) - continue; - OpsChanged |= tryToVectorize(I, R); - } - // Try to vectorize list of compares. - // Sort by type, compare predicate, etc. - auto CompareSorter = [&](Value *V, Value *V2) { - return compareCmp(V, V2, *TLI, - [&R](Instruction *I) { return R.isDeleted(I); }); - }; - - auto AreCompatibleCompares = [&](Value *V1, Value *V2) { - if (V1 == V2) - return true; - return compareCmp(V1, V2, *TLI, - [&R](Instruction *I) { return R.isDeleted(I); }); - }; - - SmallVector Vals(PostponedCmps.begin(), PostponedCmps.end()); - OpsChanged |= tryToVectorizeSequence( - Vals, CompareSorter, AreCompatibleCompares, - [this, &R](ArrayRef Candidates, bool LimitForRegisterSize) { - // Exclude possible reductions from other blocks. - bool ArePossiblyReducedInOtherBlock = - any_of(Candidates, [](Value *V) { - return any_of(V->users(), [V](User *U) { - return isa(U) && - cast(U)->getParent() != - cast(V)->getParent(); - }); - }); - if (ArePossiblyReducedInOtherBlock) - return false; - return tryToVectorizeList(Candidates, R, LimitForRegisterSize); - }, - /*LimitForRegisterSize=*/true, R); + OpsChanged |= vectorizeCmpInsts(PostponedCmps, BB, R); Instructions.clear(); } else { Instructions.clear();