138 changes: 47 additions & 91 deletions llvm/lib/CodeGen/MachineVerifier.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -573,16 +573,6 @@ void MachineVerifier::visitMachineFunctionBefore() {
verifyStackFrame();
}

// Does iterator point to a and b as the first two elements?
static bool matchPair(MachineBasicBlock::const_succ_iterator i,
const MachineBasicBlock *a, const MachineBasicBlock *b) {
if (*i == a)
return *++i == b;
if (*i == b)
return *++i == a;
return false;
}

void
MachineVerifier::visitMachineBasicBlockBefore(const MachineBasicBlock *MBB) {
FirstTerminator = nullptr;
Expand Down Expand Up @@ -616,20 +606,6 @@ MachineVerifier::visitMachineBasicBlockBefore(const MachineBasicBlock *MBB) {
}
}

// Count the number of INLINEASM_BR indirect target successors.
SmallPtrSet<const MachineBasicBlock*, 4> IndirectTargetSuccs;
for (const auto *succ : MBB->successors()) {
if (MBB->isInlineAsmBrIndirectTarget(succ))
IndirectTargetSuccs.insert(succ);
if (!FunctionBlocks.count(succ))
report("MBB has successor that isn't part of the function.", MBB);
if (!MBBInfoMap[succ].Preds.count(MBB)) {
report("Inconsistent CFG", MBB);
errs() << "MBB is not in the predecessor list of the successor "
<< printMBBReference(*succ) << ".\n";
}
}

// Check the predecessor list.
for (const MachineBasicBlock *Pred : MBB->predecessors()) {
if (!FunctionBlocks.count(Pred))
Expand Down Expand Up @@ -660,26 +636,6 @@ MachineVerifier::visitMachineBasicBlockBefore(const MachineBasicBlock *MBB) {
// check whether its answers match up with reality.
if (!TBB && !FBB) {
// Block falls through to its successor.
MachineFunction::const_iterator MBBI = std::next(MBB->getIterator());
if (MBBI == MF->end()) {
// It's possible that the block legitimately ends with a noreturn
// call or an unreachable, in which case it won't actually fall
// out the bottom of the function.
} else if (MBB->succ_size() == LandingPadSuccs.size() ||
MBB->succ_size() == IndirectTargetSuccs.size()) {
// It's possible that the block legitimately ends with a noreturn
// call or an unreachable, in which case it won't actually fall
// out of the block.
} else if ((LandingPadSuccs.size() &&
MBB->succ_size() != 1 + LandingPadSuccs.size()) ||
(IndirectTargetSuccs.size() &&
MBB->succ_size() != 1 + IndirectTargetSuccs.size())) {
report("MBB exits via unconditional fall-through but doesn't have "
"exactly one CFG successor!", MBB);
} else if (!MBB->isSuccessor(&*MBBI)) {
report("MBB exits via unconditional fall-through but its successor "
"differs from its CFG successor!", MBB);
}
if (!MBB->empty() && MBB->back().isBarrier() &&
!TII->isPredicated(MBB->back())) {
report("MBB exits via unconditional fall-through but ends with a "
Expand All @@ -691,20 +647,6 @@ MachineVerifier::visitMachineBasicBlockBefore(const MachineBasicBlock *MBB) {
}
} else if (TBB && !FBB && Cond.empty()) {
// Block unconditionally branches somewhere.
// If the block has exactly one successor, that happens to be a
// landingpad, accept it as valid control flow.
if (MBB->succ_size() != 1+LandingPadSuccs.size() &&
(MBB->succ_size() != 1 || LandingPadSuccs.size() != 1 ||
*MBB->succ_begin() != *LandingPadSuccs.begin()) &&
MBB->succ_size() != 1 + IndirectTargetSuccs.size() &&
(MBB->succ_size() != 1 || IndirectTargetSuccs.size() != 1 ||
*MBB->succ_begin() != *IndirectTargetSuccs.begin())) {
report("MBB exits via unconditional branch but doesn't have "
"exactly one CFG successor!", MBB);
} else if (!MBB->isSuccessor(TBB)) {
report("MBB exits via unconditional branch but the CFG "
"successor doesn't match the actual successor!", MBB);
}
if (MBB->empty()) {
report("MBB exits via unconditional branch but doesn't contain "
"any instructions!", MBB);
Expand All @@ -717,24 +659,6 @@ MachineVerifier::visitMachineBasicBlockBefore(const MachineBasicBlock *MBB) {
}
} else if (TBB && !FBB && !Cond.empty()) {
// Block conditionally branches somewhere, otherwise falls through.
MachineFunction::const_iterator MBBI = std::next(MBB->getIterator());
if (MBBI == MF->end()) {
report("MBB conditionally falls through out of function!", MBB);
} else if (MBB->succ_size() == 1) {
// A conditional branch with only one successor is weird, but allowed.
if (&*MBBI != TBB)
report("MBB exits via conditional branch/fall-through but only has "
"one CFG successor!", MBB);
else if (TBB != *MBB->succ_begin())
report("MBB exits via conditional branch/fall-through but the CFG "
"successor don't match the actual successor!", MBB);
} else if (MBB->succ_size() != 2) {
report("MBB exits via conditional branch/fall-through but doesn't have "
"exactly two CFG successors!", MBB);
} else if (!matchPair(MBB->succ_begin(), TBB, &*MBBI)) {
report("MBB exits via conditional branch/fall-through but the CFG "
"successors don't match the actual successors!", MBB);
}
if (MBB->empty()) {
report("MBB exits via conditional branch/fall-through but doesn't "
"contain any instructions!", MBB);
Expand All @@ -748,21 +672,6 @@ MachineVerifier::visitMachineBasicBlockBefore(const MachineBasicBlock *MBB) {
} else if (TBB && FBB) {
// Block conditionally branches somewhere, otherwise branches
// somewhere else.
if (MBB->succ_size() == 1) {
// A conditional branch with only one successor is weird, but allowed.
if (FBB != TBB)
report("MBB exits via conditional branch/branch through but only has "
"one CFG successor!", MBB);
else if (TBB != *MBB->succ_begin())
report("MBB exits via conditional branch/branch through but the CFG "
"successor don't match the actual successor!", MBB);
} else if (MBB->succ_size() != 2) {
report("MBB exits via conditional branch/branch but doesn't have "
"exactly two CFG successors!", MBB);
} else if (!matchPair(MBB->succ_begin(), TBB, FBB)) {
report("MBB exits via conditional branch/branch but the CFG "
"successors don't match the actual successors!", MBB);
}
if (MBB->empty()) {
report("MBB exits via conditional branch/branch but doesn't "
"contain any instructions!", MBB);
Expand All @@ -780,6 +689,53 @@ MachineVerifier::visitMachineBasicBlockBefore(const MachineBasicBlock *MBB) {
} else {
report("analyzeBranch returned invalid data!", MBB);
}

// Now check that the successors match up with the answers reported by
// analyzeBranch.
if (TBB && !MBB->isSuccessor(TBB))
report("MBB exits via jump or conditional branch, but its target isn't a "
"CFG successor!",
MBB);
if (FBB && !MBB->isSuccessor(FBB))
report("MBB exits via conditional branch, but its target isn't a CFG "
"successor!",
MBB);

// There might be a fallthrough to the next block if there's either no
// unconditional true branch, or if there's a condition, and one of the
// branches is missing.
bool Fallthrough = !TBB || (!Cond.empty() && !FBB);

// A conditional fallthrough must be an actual CFG successor, not
// unreachable. (Conversely, an unconditional fallthrough might not really
// be a successor, because the block might end in unreachable.)
if (!Cond.empty() && !FBB) {
MachineFunction::const_iterator MBBI = std::next(MBB->getIterator());
if (MBBI == MF->end()) {
report("MBB conditionally falls through out of function!", MBB);
} else if (!MBB->isSuccessor(&*MBBI))
report("MBB exits via conditional branch/fall-through but the CFG "
"successors don't match the actual successors!",
MBB);
}

// Verify that there aren't any extra un-accounted-for successors.
for (const MachineBasicBlock *SuccMBB : MBB->successors()) {
// If this successor is one of the branch targets, it's okay.
if (SuccMBB == TBB || SuccMBB == FBB)
continue;
// If we might have a fallthrough, and the successor is the fallthrough
// block, that's also ok.
if (Fallthrough && SuccMBB == MBB->getNextNode())
continue;
// Also accept successors which are for exception-handling or might be
// inlineasm_br targets.
if (SuccMBB->isEHPad() || MBB->isInlineAsmBrIndirectTarget(SuccMBB))
continue;
report("MBB has unexpected successors which are not branch targets, "
"fallthrough, EHPads, or inlineasm_br targets.",
MBB);
}
}

regsLive.clear();
Expand Down
11 changes: 11 additions & 0 deletions llvm/lib/CodeGen/TailDuplicator.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -813,6 +813,8 @@ bool TailDuplicator::tailDuplicate(bool IsSimple, MachineBasicBlock *TailBB,
LLVM_DEBUG(dbgs() << "\n*** Tail-duplicating " << printMBBReference(*TailBB)
<< '\n');

bool ShouldUpdateTerminators = TailBB->canFallThrough();

DenseSet<unsigned> UsedByPhi;
getRegsUsedByPHIs(*TailBB, &UsedByPhi);

Expand Down Expand Up @@ -885,6 +887,10 @@ bool TailDuplicator::tailDuplicate(bool IsSimple, MachineBasicBlock *TailBB,
for (MachineBasicBlock *Succ : TailBB->successors())
PredBB->addSuccessor(Succ, MBPI->getEdgeProbability(TailBB, Succ));

// Update branches in pred to jump to tail's layout successor if needed.
if (ShouldUpdateTerminators)
PredBB->updateTerminator(TailBB->getNextNode());

Changed = true;
++NumTailDups;
}
Expand Down Expand Up @@ -943,6 +949,11 @@ bool TailDuplicator::tailDuplicate(bool IsSimple, MachineBasicBlock *TailBB,
PrevBB->removeSuccessor(PrevBB->succ_begin());
assert(PrevBB->succ_empty());
PrevBB->transferSuccessors(TailBB);

// Update branches in PrevBB based on Tail's layout successor.
if (ShouldUpdateTerminators)
PrevBB->updateTerminator(TailBB->getNextNode());

TDBBs.push_back(PrevBB);
Changed = true;
}
Expand Down
2 changes: 0 additions & 2 deletions llvm/lib/Target/AArch64/AArch64ConditionOptimizer.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -295,8 +295,6 @@ void AArch64ConditionOptimizer::modifyCmp(MachineInstr *CmpMI,
.add(BrMI.getOperand(1));
BrMI.eraseFromParent();

MBB->updateTerminator();

++NumConditionsAdjusted;
}

Expand Down
2 changes: 1 addition & 1 deletion llvm/lib/Target/AArch64/AArch64ConditionalCompares.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -710,7 +710,7 @@ void SSACCmpConv::convert(SmallVectorImpl<MachineBasicBlock *> &RemovedBlocks) {
.add(CmpMI->getOperand(1)); // Branch target.
}
CmpMI->eraseFromParent();
Head->updateTerminator();
Head->updateTerminator(CmpBB->getNextNode());

RemovedBlocks.push_back(CmpBB);
CmpBB->eraseFromParent();
Expand Down
5 changes: 3 additions & 2 deletions llvm/lib/Target/ARM/ARMConstantIslandPass.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2361,6 +2361,7 @@ adjustJTTargetBlockForward(MachineBasicBlock *BB, MachineBasicBlock *JTBB) {
SmallVector<MachineOperand, 4> CondPrior;
MachineFunction::iterator BBi = BB->getIterator();
MachineFunction::iterator OldPrior = std::prev(BBi);
MachineFunction::iterator OldNext = std::next(BBi);

// If the block terminator isn't analyzable, don't try to move the block
bool B = TII->analyzeBranch(*BB, TBB, FBB, Cond);
Expand All @@ -2371,8 +2372,8 @@ adjustJTTargetBlockForward(MachineBasicBlock *BB, MachineBasicBlock *JTBB) {
if (!B && Cond.empty() && BB != &MF->front() &&
!TII->analyzeBranch(*OldPrior, TBB, FBB, CondPrior)) {
BB->moveAfter(JTBB);
OldPrior->updateTerminator();
BB->updateTerminator();
OldPrior->updateTerminator(BB);
BB->updateTerminator(OldNext != MF->end() ? &*OldNext : nullptr);
// Update numbering to account for the block being moved.
MF->RenumberBlocks();
++NumJTMoved;
Expand Down
6 changes: 4 additions & 2 deletions llvm/lib/Target/Hexagon/HexagonEarlyIfConv.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1017,18 +1017,20 @@ void HexagonEarlyIfConversion::mergeBlocks(MachineBasicBlock *PredB,
PredB->removeSuccessor(SuccB);
PredB->splice(PredB->end(), SuccB, SuccB->begin(), SuccB->end());
PredB->transferSuccessorsAndUpdatePHIs(SuccB);
MachineBasicBlock *OldLayoutSuccessor = SuccB->getNextNode();
removeBlock(SuccB);
if (!TermOk)
PredB->updateTerminator();
PredB->updateTerminator(OldLayoutSuccessor);
}

void HexagonEarlyIfConversion::simplifyFlowGraph(const FlowPattern &FP) {
MachineBasicBlock *OldLayoutSuccessor = FP.SplitB->getNextNode();
if (FP.TrueB)
removeBlock(FP.TrueB);
if (FP.FalseB)
removeBlock(FP.FalseB);

FP.SplitB->updateTerminator();
FP.SplitB->updateTerminator(OldLayoutSuccessor);
if (FP.SplitB->succ_size() != 1)
return;

Expand Down
15 changes: 13 additions & 2 deletions llvm/lib/Target/WebAssembly/WebAssemblyCFGSort.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -159,8 +159,16 @@ static void maybeUpdateTerminator(MachineBasicBlock *MBB) {
}
assert((AnyBarrier || AllAnalyzable) &&
"analyzeBranch needs to analyze any block with a fallthrough");

// Find the layout successor from the original block order.
MachineFunction *MF = MBB->getParent();
MachineBasicBlock *OriginalSuccessor =
unsigned(MBB->getNumber() + 1) < MF->getNumBlockIDs()
? MF->getBlockNumbered(MBB->getNumber() + 1)
: nullptr;

if (AllAnalyzable)
MBB->updateTerminator();
MBB->updateTerminator(OriginalSuccessor);
}

namespace {
Expand Down Expand Up @@ -247,9 +255,12 @@ struct Entry {
static void sortBlocks(MachineFunction &MF, const MachineLoopInfo &MLI,
const WebAssemblyExceptionInfo &WEI,
const MachineDominatorTree &MDT) {
// Remember original layout ordering, so we can update terminators after
// reordering to point to the original layout successor.
MF.RenumberBlocks();

// Prepare for a topological sort: Record the number of predecessors each
// block has, ignoring loop backedges.
MF.RenumberBlocks();
SmallVector<unsigned, 16> NumPredsLeft(MF.getNumBlockIDs(), 0);
for (MachineBasicBlock &MBB : MF) {
unsigned N = MBB.pred_size();
Expand Down
1 change: 0 additions & 1 deletion llvm/test/CodeGen/Hexagon/cext-opt-range-offset.mir
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,6 @@ body: |
successors: %bb.4
bb.4:
successors: %bb.4
%5 = A2_tfrsi -234944521
%6 = A2_tfrsi -360185632
L4_and_memopw_io %6, 0, %5
Expand Down
1 change: 1 addition & 0 deletions llvm/test/CodeGen/WebAssembly/eh-labels.mir
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ body: |
EH_LABEL <mcsymbol .Ltmp0>
CALL @foo, implicit-def dead $arguments, implicit $sp32, implicit $sp64
EH_LABEL <mcsymbol .Ltmp1>
BR %bb.2, implicit-def dead $arguments
bb.1 (landing-pad):
; predecessors: %bb.0
Expand Down
3 changes: 2 additions & 1 deletion llvm/test/MachineVerifier/verifier-pseudo-terminators.mir
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,8 @@
# Make sure that mismatched successors are caught when a _term
# instruction is used

# CHECK: *** Bad machine code: MBB exits via unconditional branch but the CFG successor doesn't match the actual successor! ***
# CHECK: *** Bad machine code: MBB exits via jump or conditional branch, but its target isn't a CFG successor! ***
# CHECK: *** Bad machine code: MBB has unexpected successors which are not branch targets, fallthrough, EHPads, or inlineasm_br targets. ***

---
name: verifier_pseudo_terminators
Expand Down