218 changes: 155 additions & 63 deletions clang/lib/Tooling/Syntax/Tokens.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -55,45 +55,140 @@ getTokensCovering(llvm::ArrayRef<syntax::Token> Toks, SourceRange R,
return {Begin, End};
}

// Finds the smallest expansion range that contains expanded tokens First and
// Last, e.g.:
// Finds the range within FID corresponding to expanded tokens [First, Last].
// Prev precedes First and Next follows Last, these must *not* be included.
// If no range satisfies the criteria, returns an invalid range.
//
// #define ID(x) x
// ID(ID(ID(a1) a2))
// ~~ -> a1
// ~~ -> a2
// ~~~~~~~~~ -> a1 a2
SourceRange findCommonRangeForMacroArgs(const syntax::Token &First,
const syntax::Token &Last,
const SourceManager &SM) {
SourceRange Res;
auto FirstLoc = First.location(), LastLoc = Last.location();
// Keep traversing up the spelling chain as longs as tokens are part of the
// same expansion.
while (!FirstLoc.isFileID() && !LastLoc.isFileID()) {
auto ExpInfoFirst = SM.getSLocEntry(SM.getFileID(FirstLoc)).getExpansion();
auto ExpInfoLast = SM.getSLocEntry(SM.getFileID(LastLoc)).getExpansion();
// Stop if expansions have diverged.
if (ExpInfoFirst.getExpansionLocStart() !=
ExpInfoLast.getExpansionLocStart())
SourceRange spelledForExpandedSlow(SourceLocation First, SourceLocation Last,
SourceLocation Prev, SourceLocation Next,
FileID TargetFile,
const SourceManager &SM) {
// There are two main parts to this algorithm:
// - identifying which spelled range covers the expanded tokens
// - validating that this range doesn't cover any extra tokens (First/Last)
//
// We do these in order. However as we transform the expanded range into the
// spelled one, we adjust First/Last so the validation remains simple.

assert(SM.getSLocEntry(TargetFile).isFile());
// In most cases, to select First and Last we must return their expansion
// range, i.e. the whole of any macros they are included in.
//
// When First and Last are part of the *same macro arg* of a macro written
// in TargetFile, we that slice of the arg, i.e. their spelling range.
//
// Unwrap such macro calls. If the target file has A(B(C)), the
// SourceLocation stack of a token inside C shows us the expansion of A first,
// then B, then any macros inside C's body, then C itself.
// (This is the reverse of the order the PP applies the expansions in).
while (First.isMacroID() && Last.isMacroID()) {
auto DecFirst = SM.getDecomposedLoc(First);
auto DecLast = SM.getDecomposedLoc(Last);
auto &ExpFirst = SM.getSLocEntry(DecFirst.first).getExpansion();
auto &ExpLast = SM.getSLocEntry(DecLast.first).getExpansion();

if (!ExpFirst.isMacroArgExpansion() || !ExpLast.isMacroArgExpansion())
break;
// Locations are in the same macro arg if they expand to the same place.
// (They may still have different FileIDs - an arg can have >1 chunks!)
if (ExpFirst.getExpansionLocStart() != ExpLast.getExpansionLocStart())
break;
// Do not continue into macro bodies.
if (!ExpInfoFirst.isMacroArgExpansion() ||
!ExpInfoLast.isMacroArgExpansion())
// Careful, given:
// #define HIDE ID(ID(a))
// ID(ID(HIDE))
// The token `a` is wrapped in 4 arg-expansions, we only want to unwrap 2.
// We distinguish them by whether the macro expands into the target file.
// Fortunately, the target file ones will always appear first.
auto &ExpMacro =
SM.getSLocEntry(SM.getFileID(ExpFirst.getExpansionLocStart()))
.getExpansion();
if (ExpMacro.getExpansionLocStart().isMacroID())
break;
FirstLoc = SM.getImmediateSpellingLoc(FirstLoc);
LastLoc = SM.getImmediateSpellingLoc(LastLoc);
// Update the result afterwards, as we want the tokens that triggered the
// expansion.
Res = {FirstLoc, LastLoc};
// Replace each endpoint with its spelling inside the macro arg.
// (This is getImmediateSpellingLoc without repeating lookups).
First = ExpFirst.getSpellingLoc().getLocWithOffset(DecFirst.second);
Last = ExpLast.getSpellingLoc().getLocWithOffset(DecLast.second);

// Now: how do we adjust the previous/next bounds? Three cases:
// A) If they are also part of the same macro arg, we translate them too.
// This will ensure that we don't select any macros nested within the
// macro arg that cover extra tokens. Critical case:
// #define ID(X) X
// ID(prev target) // selecting 'target' succeeds
// #define LARGE ID(prev target)
// LARGE // selecting 'target' fails.
// B) They are not in the macro at all, then their expansion range is a
// sibling to it, and we can safely substitute that.
// #define PREV prev
// #define ID(X) X
// PREV ID(target) // selecting 'target' succeeds.
// #define LARGE PREV ID(target)
// LARGE // selecting 'target' fails.
// C) They are in a different arg of this macro, or the macro body.
// Now selecting the whole macro arg is fine, but the whole macro is not.
// Model this by setting using the edge of the macro call as the bound.
// #define ID2(X, Y) X Y
// ID2(prev, target) // selecting 'target' succeeds
// #define LARGE ID2(prev, target)
// LARGE // selecting 'target' fails
auto AdjustBound = [&](SourceLocation &Bound) {
if (Bound.isInvalid() || !Bound.isMacroID()) // Non-macro must be case B.
return;
auto DecBound = SM.getDecomposedLoc(Bound);
auto &ExpBound = SM.getSLocEntry(DecBound.first).getExpansion();
if (ExpBound.isMacroArgExpansion() &&
ExpBound.getExpansionLocStart() == ExpFirst.getExpansionLocStart()) {
// Case A: translate to (spelling) loc within the macro arg.
Bound = ExpBound.getSpellingLoc().getLocWithOffset(DecBound.second);
return;
}
while (Bound.isMacroID()) {
SourceRange Exp = SM.getImmediateExpansionRange(Bound).getAsRange();
if (Exp.getBegin() == ExpMacro.getExpansionLocStart()) {
// Case B: bounds become the macro call itself.
Bound = (&Bound == &Prev) ? Exp.getBegin() : Exp.getEnd();
return;
}
// Either case C, or expansion location will later find case B.
// We choose the upper bound for Prev and the lower one for Next:
// ID(prev) target ID(next)
// ^ ^
// new-prev new-next
Bound = (&Bound == &Prev) ? Exp.getEnd() : Exp.getBegin();
}
};
AdjustBound(Prev);
AdjustBound(Next);
}
// Normally mapping back to expansion location here only changes FileID, as
// we've already found some tokens expanded from the same macro argument, and
// they should map to a consecutive subset of spelled tokens. Unfortunately
// SourceManager::isBeforeInTranslationUnit discriminates sourcelocations
// based on their FileID in addition to offsets. So even though we are
// referring to same tokens, SourceManager might tell us that one is before
// the other if they've got different FileIDs.
return SM.getExpansionRange(CharSourceRange(Res, true)).getAsRange();

// In all remaining cases we need the full containing macros.
// If this overlaps Prev or Next, then no range is possible.
SourceRange Candidate =
SM.getExpansionRange(SourceRange(First, Last)).getAsRange();
auto DecFirst = SM.getDecomposedExpansionLoc(Candidate.getBegin());
auto DecLast = SM.getDecomposedLoc(Candidate.getEnd());
// Can end up in the wrong file due to bad input or token-pasting shenanigans.
if (Candidate.isInvalid() || DecFirst.first != TargetFile || DecLast.first != TargetFile)
return SourceRange();
// Check bounds, which may still be inside macros.
if (Prev.isValid()) {
auto Dec = SM.getDecomposedLoc(SM.getExpansionRange(Prev).getBegin());
if (Dec.first != DecFirst.first || Dec.second >= DecFirst.second)
return SourceRange();
}
if (Next.isValid()) {
auto Dec = SM.getDecomposedLoc(SM.getExpansionRange(Next).getEnd());
if (Dec.first != DecLast.first || Dec.second <= DecLast.second)
return SourceRange();
}
// Now we know that Candidate is a file range that covers [First, Last]
// without encroaching on {Prev, Next}. Ship it!
return Candidate;
}

} // namespace
Expand Down Expand Up @@ -363,51 +458,48 @@ TokenBuffer::spelledForExpanded(llvm::ArrayRef<syntax::Token> Expanded) const {
// of the range, bail out in that case.
if (Expanded.empty())
return llvm::None;
const syntax::Token *First = &Expanded.front();
const syntax::Token *Last = &Expanded.back();
auto [FirstSpelled, FirstMapping] = spelledForExpandedToken(First);
auto [LastSpelled, LastMapping] = spelledForExpandedToken(Last);

const syntax::Token *BeginSpelled;
const Mapping *BeginMapping;
std::tie(BeginSpelled, BeginMapping) =
spelledForExpandedToken(&Expanded.front());

const syntax::Token *LastSpelled;
const Mapping *LastMapping;
std::tie(LastSpelled, LastMapping) =
spelledForExpandedToken(&Expanded.back());

FileID FID = SourceMgr->getFileID(BeginSpelled->location());
FileID FID = SourceMgr->getFileID(FirstSpelled->location());
// FIXME: Handle multi-file changes by trying to map onto a common root.
if (FID != SourceMgr->getFileID(LastSpelled->location()))
return llvm::None;

const MarkedFile &File = Files.find(FID)->second;

// If both tokens are coming from a macro argument expansion, try and map to
// smallest part of the macro argument. BeginMapping && LastMapping check is
// only for performance, they are a prerequisite for Expanded.front() and
// Expanded.back() being part of a macro arg expansion.
if (BeginMapping && LastMapping &&
SourceMgr->isMacroArgExpansion(Expanded.front().location()) &&
SourceMgr->isMacroArgExpansion(Expanded.back().location())) {
auto CommonRange = findCommonRangeForMacroArgs(Expanded.front(),
Expanded.back(), *SourceMgr);
// It might be the case that tokens are arguments of different macro calls,
// in that case we should continue with the logic below instead of returning
// an empty range.
if (CommonRange.isValid())
return getTokensCovering(File.SpelledTokens, CommonRange, *SourceMgr);
// If the range is within one macro argument, the result may be only part of a
// Mapping. We must use the general (SourceManager-based) algorithm.
if (FirstMapping && FirstMapping == LastMapping &&
SourceMgr->isMacroArgExpansion(First->location()) &&
SourceMgr->isMacroArgExpansion(Last->location())) {
// We use excluded Prev/Next token for bounds checking.
SourceLocation Prev = (First == &ExpandedTokens.front())
? SourceLocation()
: (First - 1)->location();
SourceLocation Next = (Last == &ExpandedTokens.back())
? SourceLocation()
: (Last + 1)->location();
SourceRange Range = spelledForExpandedSlow(
First->location(), Last->location(), Prev, Next, FID, *SourceMgr);
if (Range.isInvalid())
return llvm::None;
return getTokensCovering(File.SpelledTokens, Range, *SourceMgr);
}

// Otherwise, use the fast version based on Mappings.
// Do not allow changes that doesn't cover full expansion.
unsigned BeginExpanded = Expanded.begin() - ExpandedTokens.data();
unsigned EndExpanded = Expanded.end() - ExpandedTokens.data();
if (BeginMapping && BeginExpanded != BeginMapping->BeginExpanded)
unsigned FirstExpanded = Expanded.begin() - ExpandedTokens.data();
unsigned LastExpanded = Expanded.end() - ExpandedTokens.data();
if (FirstMapping && FirstExpanded != FirstMapping->BeginExpanded)
return llvm::None;
if (LastMapping && LastMapping->EndExpanded != EndExpanded)
if (LastMapping && LastMapping->EndExpanded != LastExpanded)
return llvm::None;
// All is good, return the result.
return llvm::makeArrayRef(
BeginMapping ? File.SpelledTokens.data() + BeginMapping->BeginSpelled
: BeginSpelled,
FirstMapping ? File.SpelledTokens.data() + FirstMapping->BeginSpelled
: FirstSpelled,
LastMapping ? File.SpelledTokens.data() + LastMapping->EndSpelled
: LastSpelled + 1);
}
Expand Down
56 changes: 56 additions & 0 deletions clang/unittests/Tooling/Syntax/TokensTest.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -743,6 +743,62 @@ TEST_F(TokenBufferTest, SpelledByExpanded) {
ValueIs(SameRange(findSpelled("ID2 ( a4 , a5 a6 a7 )"))));
// Should fail, spans multiple invocations.
EXPECT_EQ(Buffer.spelledForExpanded(findExpanded("a1 a2 a3 a4")), llvm::None);

// https://github.com/clangd/clangd/issues/1289
recordTokens(R"cpp(
#define FOO(X) foo(X)
#define INDIRECT FOO(y)
INDIRECT // expands to foo(y)
)cpp");
EXPECT_EQ(Buffer.spelledForExpanded(findExpanded("y")), llvm::None);

recordTokens(R"cpp(
#define FOO(X) a X b
FOO(y)
)cpp");
EXPECT_THAT(Buffer.spelledForExpanded(findExpanded("y")),
ValueIs(SameRange(findSpelled("y"))));

recordTokens(R"cpp(
#define ID(X) X
#define BAR ID(1)
BAR
)cpp");
EXPECT_THAT(Buffer.spelledForExpanded(findExpanded("1")),
ValueIs(SameRange(findSpelled(") BAR").drop_front())));

// Critical cases for mapping of Prev/Next in spelledForExpandedSlow.
recordTokens(R"cpp(
#define ID(X) X
ID(prev ID(good))
#define LARGE ID(prev ID(bad))
LARGE
)cpp");
EXPECT_THAT(Buffer.spelledForExpanded(findExpanded("good")),
ValueIs(SameRange(findSpelled("good"))));
EXPECT_EQ(Buffer.spelledForExpanded(findExpanded("bad")), llvm::None);

recordTokens(R"cpp(
#define PREV prev
#define ID(X) X
PREV ID(good)
#define LARGE PREV ID(bad)
LARGE
)cpp");
EXPECT_THAT(Buffer.spelledForExpanded(findExpanded("good")),
ValueIs(SameRange(findSpelled("good"))));
EXPECT_EQ(Buffer.spelledForExpanded(findExpanded("bad")), llvm::None);

recordTokens(R"cpp(
#define ID(X) X
#define ID2(X, Y) X Y
ID2(prev, ID(good))
#define LARGE ID2(prev, bad)
LARGE
)cpp");
EXPECT_THAT(Buffer.spelledForExpanded(findExpanded("good")),
ValueIs(SameRange(findSpelled("good"))));
EXPECT_EQ(Buffer.spelledForExpanded(findExpanded("bad")), llvm::None);
}

TEST_F(TokenBufferTest, ExpandedTokensForRange) {
Expand Down
10 changes: 5 additions & 5 deletions llvm/lib/Support/X86TargetParser.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -203,10 +203,10 @@ constexpr FeatureBitset FeaturesTigerlake =
FeatureCLWB | FeatureMOVDIRI | FeatureSHSTK | FeatureKL | FeatureWIDEKL;
constexpr FeatureBitset FeaturesSapphireRapids =
FeaturesICLServer | FeatureAMX_BF16 | FeatureAMX_INT8 | FeatureAMX_TILE |
FeatureAVX512BF16 | FeatureAVX512FP16 | FeatureAVX512VP2INTERSECT |
FeatureAVXVNNI | FeatureCLDEMOTE | FeatureENQCMD | FeatureMOVDIR64B |
FeatureMOVDIRI | FeaturePTWRITE | FeatureSERIALIZE | FeatureSHSTK |
FeatureTSXLDTRK | FeatureUINTR | FeatureWAITPKG;
FeatureAVX512BF16 | FeatureAVX512FP16 | FeatureAVXVNNI | FeatureCLDEMOTE |
FeatureENQCMD | FeatureMOVDIR64B | FeatureMOVDIRI | FeaturePTWRITE |
FeatureSERIALIZE | FeatureSHSTK | FeatureTSXLDTRK | FeatureUINTR |
FeatureWAITPKG;

// Intel Atom processors.
// Bonnell has feature parity with Core2 and adds MOVBE.
Expand Down Expand Up @@ -367,7 +367,7 @@ constexpr ProcInfo Processors[] = {
// Tigerlake microarchitecture based processors.
{ {"tigerlake"}, CK_Tigerlake, FEATURE_AVX512VP2INTERSECT, FeaturesTigerlake },
// Sapphire Rapids microarchitecture based processors.
{ {"sapphirerapids"}, CK_SapphireRapids, FEATURE_AVX512VP2INTERSECT, FeaturesSapphireRapids },
{ {"sapphirerapids"}, CK_SapphireRapids, FEATURE_AVX512BF16, FeaturesSapphireRapids },
// Alderlake microarchitecture based processors.
{ {"alderlake"}, CK_Alderlake, FEATURE_AVX2, FeaturesAlderlake },
// Knights Landing processor.
Expand Down
1 change: 0 additions & 1 deletion llvm/lib/Target/X86/X86.td
Original file line number Diff line number Diff line change
Expand Up @@ -909,7 +909,6 @@ def ProcessorFeatures {
FeatureTSXLDTRK,
FeatureENQCMD,
FeatureSHSTK,
FeatureVP2INTERSECT,
FeatureMOVDIRI,
FeatureMOVDIR64B,
FeatureUINTR];
Expand Down
4 changes: 3 additions & 1 deletion llvm/lib/Transforms/Utils/LoopVersioning.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -137,8 +137,10 @@ void LoopVersioning::addPHINodes(
// See if we have a single-operand PHI with the value defined by the
// original loop.
for (auto I = PHIBlock->begin(); (PN = dyn_cast<PHINode>(I)); ++I) {
if (PN->getIncomingValue(0) == Inst)
if (PN->getIncomingValue(0) == Inst) {
SE->forgetValue(PN);
break;
}
}
// If not create it.
if (!PN) {
Expand Down
125 changes: 125 additions & 0 deletions llvm/test/Transforms/LoopLoadElim/versioning-scev-invalidation.ll
Original file line number Diff line number Diff line change
@@ -0,0 +1,125 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
; RUN: opt -force-vector-width=4 -force-vector-interleave=1 -passes='loop-vectorize,loop-load-elim' -S %s | FileCheck %s

@glob.1 = external global [100 x double]
@glob.2 = external global [100 x double]

define void @g(ptr %dst.1, ptr %start, i64 %N) {
; CHECK-LABEL: @g(
; CHECK-NEXT: loop.1.lver.check:
; CHECK-NEXT: [[TMP0:%.*]] = shl i64 [[N:%.*]], 3
; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[TMP0]], 16
; CHECK-NEXT: [[UGLYGEP:%.*]] = getelementptr i8, ptr @glob.2, i64 [[TMP1]]
; CHECK-NEXT: [[UGLYGEP2:%.*]] = getelementptr i8, ptr [[DST_1:%.*]], i64 8
; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr @glob.2, [[UGLYGEP2]]
; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[DST_1]], [[UGLYGEP]]
; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
; CHECK-NEXT: br i1 [[FOUND_CONFLICT]], label [[LOOP_1_PH_LVER_ORIG:%.*]], label [[LOOP_1_PH:%.*]]
; CHECK: loop.1.ph.lver.orig:
; CHECK-NEXT: br label [[LOOP_1_LVER_ORIG:%.*]]
; CHECK: loop.1.lver.orig:
; CHECK-NEXT: [[IV_LVER_ORIG:%.*]] = phi i64 [ 0, [[LOOP_1_PH_LVER_ORIG]] ], [ [[IV_NEXT_LVER_ORIG:%.*]], [[LOOP_1_LVER_ORIG]] ]
; CHECK-NEXT: [[PTR_IV_1_LVER_ORIG:%.*]] = phi ptr [ @glob.1, [[LOOP_1_PH_LVER_ORIG]] ], [ [[PTR_IV_1_NEXT_LVER_ORIG:%.*]], [[LOOP_1_LVER_ORIG]] ]
; CHECK-NEXT: [[GEP_IV_LVER_ORIG:%.*]] = getelementptr inbounds double, ptr @glob.2, i64 [[IV_LVER_ORIG]]
; CHECK-NEXT: [[L_1_LVER_ORIG:%.*]] = load double, ptr [[GEP_IV_LVER_ORIG]], align 8
; CHECK-NEXT: [[GEP_IV_1_LVER_ORIG:%.*]] = getelementptr inbounds double, ptr getelementptr inbounds (double, ptr @glob.2, i64 1), i64 [[IV_LVER_ORIG]]
; CHECK-NEXT: store double 0.000000e+00, ptr [[GEP_IV_1_LVER_ORIG]], align 8
; CHECK-NEXT: store double 0.000000e+00, ptr [[DST_1]], align 8
; CHECK-NEXT: [[PTR_IV_1_NEXT_LVER_ORIG]] = getelementptr inbounds double, ptr [[PTR_IV_1_LVER_ORIG]], i64 1
; CHECK-NEXT: [[IV_NEXT_LVER_ORIG]] = add nuw nsw i64 [[IV_LVER_ORIG]], 1
; CHECK-NEXT: [[EXITCOND_NOT_LVER_ORIG:%.*]] = icmp eq i64 [[IV_LVER_ORIG]], [[N]]
; CHECK-NEXT: br i1 [[EXITCOND_NOT_LVER_ORIG]], label [[LOOP_2_PH_LOOPEXIT:%.*]], label [[LOOP_1_LVER_ORIG]]
; CHECK: loop.1.ph:
; CHECK-NEXT: [[LOAD_INITIAL:%.*]] = load double, ptr @glob.2, align 8
; CHECK-NEXT: br label [[LOOP_1:%.*]]
; CHECK: loop.1:
; CHECK-NEXT: [[STORE_FORWARDED:%.*]] = phi double [ [[LOAD_INITIAL]], [[LOOP_1_PH]] ], [ 0.000000e+00, [[LOOP_1]] ]
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, [[LOOP_1_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP_1]] ]
; CHECK-NEXT: [[PTR_IV_1:%.*]] = phi ptr [ @glob.1, [[LOOP_1_PH]] ], [ [[PTR_IV_1_NEXT:%.*]], [[LOOP_1]] ]
; CHECK-NEXT: [[GEP_IV:%.*]] = getelementptr inbounds double, ptr @glob.2, i64 [[IV]]
; CHECK-NEXT: [[L_1:%.*]] = load double, ptr [[GEP_IV]], align 8
; CHECK-NEXT: [[GEP_IV_1:%.*]] = getelementptr inbounds double, ptr getelementptr inbounds (double, ptr @glob.2, i64 1), i64 [[IV]]
; CHECK-NEXT: store double 0.000000e+00, ptr [[GEP_IV_1]], align 8
; CHECK-NEXT: store double 0.000000e+00, ptr [[DST_1]], align 8
; CHECK-NEXT: [[PTR_IV_1_NEXT]] = getelementptr inbounds double, ptr [[PTR_IV_1]], i64 1
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV]], [[N]]
; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label [[LOOP_2_PH_LOOPEXIT3:%.*]], label [[LOOP_1]]
; CHECK: loop.2.ph.loopexit:
; CHECK-NEXT: [[LCSSA_PTR_IV_1_PH:%.*]] = phi ptr [ [[PTR_IV_1_LVER_ORIG]], [[LOOP_1_LVER_ORIG]] ]
; CHECK-NEXT: br label [[LOOP_2_PH:%.*]]
; CHECK: loop.2.ph.loopexit3:
; CHECK-NEXT: [[LCSSA_PTR_IV_1_PH4:%.*]] = phi ptr [ [[PTR_IV_1]], [[LOOP_1]] ]
; CHECK-NEXT: br label [[LOOP_2_PH]]
; CHECK: loop.2.ph:
; CHECK-NEXT: [[LCSSA_PTR_IV_1:%.*]] = phi ptr [ [[LCSSA_PTR_IV_1_PH]], [[LOOP_2_PH_LOOPEXIT]] ], [ [[LCSSA_PTR_IV_1_PH4]], [[LOOP_2_PH_LOOPEXIT3]] ]
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
; CHECK: vector.ph:
; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; CHECK-NEXT: [[TMP2:%.*]] = mul i64 [[N_VEC]], 8
; CHECK-NEXT: [[IND_END:%.*]] = getelementptr i8, ptr [[LCSSA_PTR_IV_1]], i64 [[TMP2]]
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
; CHECK: vector.body:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP3:%.*]] = add i64 [[INDEX]], 0
; CHECK-NEXT: [[TMP4:%.*]] = mul i64 [[TMP3]], 8
; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[LCSSA_PTR_IV_1]], i64 [[TMP4]]
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr double, ptr [[NEXT_GEP]], i32 0
; CHECK-NEXT: store <4 x double> zeroinitializer, ptr [[TMP5]], align 8
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: middle.block:
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
; CHECK: scalar.ph:
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[LOOP_2_PH]] ]
; CHECK-NEXT: [[BC_RESUME_VAL1:%.*]] = phi ptr [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[LCSSA_PTR_IV_1]], [[LOOP_2_PH]] ]
; CHECK-NEXT: br label [[LOOP_2:%.*]]
; CHECK: loop.2:
; CHECK-NEXT: [[IV_2:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_2_NEXT:%.*]], [[LOOP_2]] ]
; CHECK-NEXT: [[PTR_IV_2:%.*]] = phi ptr [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ], [ [[PTR_IV_2_NEXT:%.*]], [[LOOP_2]] ]
; CHECK-NEXT: store double 0.000000e+00, ptr [[PTR_IV_2]], align 8
; CHECK-NEXT: [[PTR_IV_2_NEXT]] = getelementptr inbounds double, ptr [[PTR_IV_2]], i64 1
; CHECK-NEXT: [[IV_2_NEXT]] = add nuw nsw i64 [[IV_2]], 1
; CHECK-NEXT: [[EXITCOND_1_NOT:%.*]] = icmp eq i64 [[IV_2_NEXT]], [[N]]
; CHECK-NEXT: br i1 [[EXITCOND_1_NOT]], label [[EXIT_LOOPEXIT:%.*]], label [[LOOP_2]], !llvm.loop [[LOOP2:![0-9]+]]
; CHECK: exit.loopexit:
; CHECK-NEXT: br label [[EXIT]]
; CHECK: exit:
; CHECK-NEXT: ret void
;
entry:
br label %loop.1

loop.1:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.1 ]
%ptr.iv.1 = phi ptr [ @glob.1, %entry ], [ %ptr.iv.1.next, %loop.1 ]
%gep.iv = getelementptr inbounds double, ptr @glob.2, i64 %iv
%l.1 = load double, ptr %gep.iv, align 8
%gep.iv.1 = getelementptr inbounds double, ptr getelementptr inbounds (double, ptr @glob.2, i64 1), i64 %iv
store double 0.000000e+00, ptr %gep.iv.1, align 8
store double 0.000000e+00, ptr %dst.1, align 8
%ptr.iv.1.next = getelementptr inbounds double, ptr %ptr.iv.1, i64 1
%iv.next = add nuw nsw i64 %iv, 1
%exitcond.not = icmp eq i64 %iv, %N
br i1 %exitcond.not, label %loop.2.ph, label %loop.1

loop.2.ph:
%lcssa.ptr.iv.1 = phi ptr [ %ptr.iv.1, %loop.1 ]
br label %loop.2

loop.2:
%iv.2 = phi i64 [ 0, %loop.2.ph ] , [ %iv.2.next, %loop.2 ]
%ptr.iv.2 = phi ptr [ %lcssa.ptr.iv.1, %loop.2.ph ], [ %ptr.iv.2.next, %loop.2 ]
store double 0.000000e+00, ptr %ptr.iv.2, align 8
%ptr.iv.2.next = getelementptr inbounds double, ptr %ptr.iv.2, i64 1
%iv.2.next = add nuw nsw i64 %iv.2, 1
%exitcond.1.not = icmp eq i64 %iv.2.next, %N
br i1 %exitcond.1.not, label %exit, label %loop.2

exit:
ret void
}