diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp index 7296dba13aaac..7999f469be928 100644 --- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp +++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp @@ -2764,6 +2764,7 @@ class BoUpSLP { ScheduleEnd = nullptr; FirstLoadStoreInRegion = nullptr; LastLoadStoreInRegion = nullptr; + RegionHasStackSave = false; // Reduce the maximum schedule region size by the size of the // previous scheduling run. @@ -3032,6 +3033,11 @@ class BoUpSLP { /// (can be null). ScheduleData *LastLoadStoreInRegion = nullptr; + /// Is there an llvm.stacksave or llvm.stackrestore in the scheduling + /// region? Used to optimize the dependence calculation for the + /// common case where there isn't. + bool RegionHasStackSave = false; + /// The current size of the scheduling region. int ScheduleRegionSize = 0; @@ -8016,6 +8022,10 @@ void BoUpSLP::BlockScheduling::initScheduleData(Instruction *FromI, } CurrentLoadStore = SD; } + + if (match(I, m_Intrinsic()) || + match(I, m_Intrinsic())) + RegionHasStackSave = true; } if (NextLoadStore) { if (CurrentLoadStore) @@ -8099,40 +8109,42 @@ void BoUpSLP::BlockScheduling::calculateDependencies(ScheduleData *SD, } } - // If we have an inalloc alloca instruction, it needs to be scheduled - // after any preceeding stacksave. We also need to prevent any alloca - // from reordering above a preceeding stackrestore. - if (match(BundleMember->Inst, m_Intrinsic()) || - match(BundleMember->Inst, m_Intrinsic())) { - for (Instruction *I = BundleMember->Inst->getNextNode(); - I != ScheduleEnd; I = I->getNextNode()) { - if (match(I, m_Intrinsic()) || - match(I, m_Intrinsic())) - // Any allocas past here must be control dependent on I, and I - // must be memory dependend on BundleMember->Inst. - break; + if (RegionHasStackSave) { + // If we have an inalloc alloca instruction, it needs to be scheduled + // after any preceeding stacksave. We also need to prevent any alloca + // from reordering above a preceeding stackrestore. + if (match(BundleMember->Inst, m_Intrinsic()) || + match(BundleMember->Inst, m_Intrinsic())) { + for (Instruction *I = BundleMember->Inst->getNextNode(); + I != ScheduleEnd; I = I->getNextNode()) { + if (match(I, m_Intrinsic()) || + match(I, m_Intrinsic())) + // Any allocas past here must be control dependent on I, and I + // must be memory dependend on BundleMember->Inst. + break; - if (!isa(I)) - continue; + if (!isa(I)) + continue; - // Add the dependency - makeControlDependent(I); + // Add the dependency + makeControlDependent(I); + } } - } - // In addition to the cases handle just above, we need to prevent - // allocas from moving below a stacksave. The stackrestore case - // is currently thought to be conservatism. - if (isa(BundleMember->Inst)) { - for (Instruction *I = BundleMember->Inst->getNextNode(); - I != ScheduleEnd; I = I->getNextNode()) { - if (!match(I, m_Intrinsic()) && - !match(I, m_Intrinsic())) - continue; + // In addition to the cases handle just above, we need to prevent + // allocas from moving below a stacksave. The stackrestore case + // is currently thought to be conservatism. + if (isa(BundleMember->Inst)) { + for (Instruction *I = BundleMember->Inst->getNextNode(); + I != ScheduleEnd; I = I->getNextNode()) { + if (!match(I, m_Intrinsic()) && + !match(I, m_Intrinsic())) + continue; - // Add the dependency - makeControlDependent(I); - break; + // Add the dependency + makeControlDependent(I); + break; + } } }