diff --git a/llvm/include/llvm/ADT/GenericConvergenceVerifier.h b/llvm/include/llvm/ADT/GenericConvergenceVerifier.h new file mode 100644 index 0000000000000..213d77593c996 --- /dev/null +++ b/llvm/include/llvm/ADT/GenericConvergenceVerifier.h @@ -0,0 +1,75 @@ +//===- GenericConvergenceVerifier.h ---------------------------*- C++ -*---===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// +/// \file +/// +/// A verifer for the static rules of convergence control tokens that works with +/// both LLVM IR and MIR. +/// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_ADT_GENERICCONVERGENCEVERIFIER_H +#define LLVM_ADT_GENERICCONVERGENCEVERIFIER_H + +#include "llvm/ADT/GenericCycleInfo.h" + +namespace llvm { + +template class GenericConvergenceVerifier { +public: + using BlockT = typename ContextT::BlockT; + using FunctionT = typename ContextT::FunctionT; + using ValueRefT = typename ContextT::ValueRefT; + using InstructionT = typename ContextT::InstructionT; + using DominatorTreeT = typename ContextT::DominatorTreeT; + using CycleInfoT = GenericCycleInfo; + using CycleT = typename CycleInfoT::CycleT; + + void initialize(raw_ostream &OS, + function_ref FailureCB, + const FunctionT &F) { + clear(); + this->OS = &OS; + this->FailureCB = FailureCB; + Context = ContextT(&F); + } + + void clear(); + void visit(const InstructionT &I); + void verify(const DominatorTreeT &DT); + + bool sawTokens() const { return ConvergenceKind == ControlledConvergence; } + +private: + raw_ostream *OS; + std::function FailureCB; + DominatorTreeT *DT; + CycleInfoT CI; + ContextT Context; + + /// Whether the current function has convergencectrl operand bundles. + enum { + ControlledConvergence, + UncontrolledConvergence, + NoConvergence + } ConvergenceKind = NoConvergence; + + // Cache token uses found so far. Note that we track the unique definitions + // and not the token values. + DenseMap Tokens; + + const InstructionT *findAndCheckConvergenceTokenUsed(const InstructionT &I); + bool isControlledConvergent(const InstructionT &I); + bool isConvergent(const InstructionT &I) const; + + void reportFailure(const Twine &Message, ArrayRef Values); +}; + +} // end namespace llvm + +#endif // LLVM_ADT_GENERICCONVERGENCEVERIFIER_H diff --git a/llvm/include/llvm/ADT/GenericConvergenceVerifierImpl.h b/llvm/include/llvm/ADT/GenericConvergenceVerifierImpl.h new file mode 100644 index 0000000000000..5a50ddf256ca6 --- /dev/null +++ b/llvm/include/llvm/ADT/GenericConvergenceVerifierImpl.h @@ -0,0 +1,203 @@ +//===- GenericConvergenceVerifierImpl.h -----------------------*- C++ -*---===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// +/// \file +/// +/// A verifer for the static rules of convergence control tokens that works with +/// both LLVM IR and MIR. +/// +/// This template implementation resides in a separate file so that it does not +/// get injected into every .cpp file that includes the generic header. +/// +/// DO NOT INCLUDE THIS FILE WHEN MERELY USING CYCLEINFO. +/// +/// This file should only be included by files that implement a +/// specialization of the relevant templates. Currently these are: +/// - llvm/lib/IR/Verifier.cpp +/// - llvm/lib/CodeGen/MachineVerifier.cpp +/// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_ADT_GENERICCONVERGENCEVERIFIERIMPL_H +#define LLVM_ADT_GENERICCONVERGENCEVERIFIERIMPL_H + +#include "llvm/ADT/GenericConvergenceVerifier.h" +#include "llvm/ADT/GenericCycleInfo.h" +#include "llvm/ADT/GenericSSAContext.h" +#include "llvm/ADT/PostOrderIterator.h" +#include "llvm/ADT/Twine.h" +#include "llvm/IR/Intrinsics.h" + +using namespace llvm; + +#define Check(C, ...) \ + do { \ + if (!(C)) { \ + reportFailure(__VA_ARGS__); \ + return; \ + } \ + } while (false) + +#define CheckOrNull(C, ...) \ + do { \ + if (!(C)) { \ + reportFailure(__VA_ARGS__); \ + return {}; \ + } \ + } while (false) + +static bool isConvergenceControlIntrinsic(unsigned IntrinsicID) { + switch (IntrinsicID) { + default: + return false; + case Intrinsic::experimental_convergence_anchor: + case Intrinsic::experimental_convergence_entry: + case Intrinsic::experimental_convergence_loop: + return true; + } +} + +namespace llvm { +template void GenericConvergenceVerifier::clear() { + Tokens.clear(); + CI.clear(); + ConvergenceKind = NoConvergence; +} + +template +void GenericConvergenceVerifier::visit(const InstructionT &I) { + if (isControlledConvergent(I)) { + Check(isConvergent(I), + "Expected convergent attribute on a controlled convergent call.", + {Context.print(&I)}); + Check(ConvergenceKind != UncontrolledConvergence, + "Cannot mix controlled and uncontrolled convergence in the same " + "function.", + {Context.print(&I)}); + ConvergenceKind = ControlledConvergence; + } else if (isConvergent(I)) { + Check(ConvergenceKind != ControlledConvergence, + "Cannot mix controlled and uncontrolled convergence in the same " + "function.", + {Context.print(&I)}); + ConvergenceKind = UncontrolledConvergence; + } +} + +template +void GenericConvergenceVerifier::reportFailure( + const Twine &Message, ArrayRef DumpedValues) { + FailureCB(Message); + for (auto V : DumpedValues) { + *OS << V << '\n'; + } +} + +template +void GenericConvergenceVerifier::verify(const DominatorTreeT &DT) { + assert(Context.getFunction()); + const auto &F = *Context.getFunction(); + + DenseMap> LiveTokenMap; + DenseMap CycleHearts; + + // Just like the DominatorTree, compute the CycleInfo locally so that we + // can run the verifier outside of a pass manager and we don't rely on + // potentially out-dated analysis results. + CI.compute(const_cast(F)); + + auto checkToken = [&](const InstructionT *Token, const InstructionT *User, + SmallVectorImpl &LiveTokens) { + Check(llvm::is_contained(LiveTokens, Token), + "Convergence region is not well-nested.", + {Context.print(Token), Context.print(User)}); + while (LiveTokens.back() != Token) + LiveTokens.pop_back(); + + // Check static rules about cycles. + auto *BB = User->getParent(); + auto *BBCycle = CI.getCycle(BB); + if (!BBCycle) + return; + + auto *DefBB = Token->getParent(); + if (DefBB == BB || BBCycle->contains(DefBB)) { + // degenerate occurrence of a loop intrinsic + return; + } + + Check(ContextT::getIntrinsicID(*User) == + Intrinsic::experimental_convergence_loop, + "Convergence token used by an instruction other than " + "llvm.experimental.convergence.loop in a cycle that does " + "not contain the token's definition.", + {Context.print(User), CI.print(BBCycle)}); + + while (true) { + auto *Parent = BBCycle->getParentCycle(); + if (!Parent || Parent->contains(DefBB)) + break; + BBCycle = Parent; + }; + + Check(BBCycle->isReducible() && BB == BBCycle->getHeader(), + "Cycle heart must dominate all blocks in the cycle.", + {Context.print(User), Context.printAsOperand(BB), CI.print(BBCycle)}); + Check(!CycleHearts.count(BBCycle), + "Two static convergence token uses in a cycle that does " + "not contain either token's definition.", + {Context.print(User), Context.print(CycleHearts[BBCycle]), + CI.print(BBCycle)}); + CycleHearts[BBCycle] = User; + }; + + ReversePostOrderTraversal RPOT(&F); + SmallVector LiveTokens; + for (auto *BB : RPOT) { + LiveTokens.clear(); + auto LTIt = LiveTokenMap.find(BB); + if (LTIt != LiveTokenMap.end()) { + LiveTokens = std::move(LTIt->second); + LiveTokenMap.erase(LTIt); + } + + for (auto &I : *BB) { + if (auto *Token = Tokens.lookup(&I)) + checkToken(Token, &I, LiveTokens); + if (isConvergenceControlIntrinsic(ContextT::getIntrinsicID(I))) + LiveTokens.push_back(&I); + } + + // Propagate token liveness + for (auto *Succ : successors(BB)) { + auto *SuccNode = DT.getNode(Succ); + auto LTIt = LiveTokenMap.find(Succ); + if (LTIt == LiveTokenMap.end()) { + // We're the first predecessor: all tokens which dominate the + // successor are live for now. + LTIt = LiveTokenMap.try_emplace(Succ).first; + for (auto LiveToken : LiveTokens) { + if (!DT.dominates(DT.getNode(LiveToken->getParent()), SuccNode)) + break; + LTIt->second.push_back(LiveToken); + } + } else { + // Compute the intersection of live tokens. + auto It = llvm::partition( + LTIt->second, [&LiveTokens](const InstructionT *Token) { + return llvm::is_contained(LiveTokens, Token); + }); + LTIt->second.erase(It, LTIt->second.end()); + } + } + } +} + +} // end namespace llvm + +#endif // LLVM_ADT_GENERICCONVERGENCEVERIFIERIMPL_H diff --git a/llvm/include/llvm/ADT/GenericSSAContext.h b/llvm/include/llvm/ADT/GenericSSAContext.h index 45f5dc7d3fcd1..6aa3a8b9b6e0b 100644 --- a/llvm/include/llvm/ADT/GenericSSAContext.h +++ b/llvm/include/llvm/ADT/GenericSSAContext.h @@ -24,6 +24,10 @@ namespace llvm { template class DominatorTreeBase; template class SmallVectorImpl; +namespace Intrinsic { +typedef unsigned ID; +} + // Specializations of this template should provide the types used by the // template GenericSSAContext below. template struct GenericSSATraits; @@ -78,6 +82,8 @@ template class GenericSSAContext { const FunctionT *getFunction() const { return F; } + static Intrinsic::ID getIntrinsicID(const InstructionT &I); + static void appendBlockDefs(SmallVectorImpl &defs, BlockT &block); static void appendBlockDefs(SmallVectorImpl &defs, const BlockT &block); @@ -91,6 +97,7 @@ template class GenericSSAContext { const BlockT *getDefBlock(ConstValueRefT value) const; Printable print(const BlockT *block) const; + Printable printAsOperand(const BlockT *BB) const; Printable print(const InstructionT *inst) const; Printable print(ConstValueRefT value) const; }; diff --git a/llvm/include/llvm/IR/ConvergenceVerifier.h b/llvm/include/llvm/IR/ConvergenceVerifier.h new file mode 100644 index 0000000000000..df4c495f6a66f --- /dev/null +++ b/llvm/include/llvm/IR/ConvergenceVerifier.h @@ -0,0 +1,28 @@ +//===- ConvergenceVerifier.h - Verify convergenctrl -------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// \file +/// +/// This file declares the LLVM IR specialization of the +/// GenericConvergenceVerifier template. +/// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_IR_CONVERGENCEVERIFIER_H +#define LLVM_IR_CONVERGENCEVERIFIER_H + +#include "llvm/ADT/GenericConvergenceVerifier.h" +#include "llvm/IR/SSAContext.h" + +namespace llvm { + +extern template class GenericConvergenceVerifier; +using ConvergenceVerifier = GenericConvergenceVerifier; + +} // namespace llvm + +#endif // LLVM_IR_CONVERGENCEVERIFIER_H diff --git a/llvm/lib/CodeGen/MachineSSAContext.cpp b/llvm/lib/CodeGen/MachineSSAContext.cpp index 4311255ed76d0..e384187b6e859 100644 --- a/llvm/lib/CodeGen/MachineSSAContext.cpp +++ b/llvm/lib/CodeGen/MachineSSAContext.cpp @@ -13,6 +13,7 @@ //===----------------------------------------------------------------------===// #include "llvm/CodeGen/MachineSSAContext.h" +#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h" #include "llvm/CodeGen/MachineBasicBlock.h" #include "llvm/CodeGen/MachineFunction.h" #include "llvm/CodeGen/MachineInstr.h" @@ -58,6 +59,13 @@ bool MachineSSAContext::isConstantOrUndefValuePhi(const MachineInstr &Phi) { return Phi.isConstantValuePHI(); } +template <> +Intrinsic::ID MachineSSAContext::getIntrinsicID(const MachineInstr &MI) { + if (auto *GI = dyn_cast(&MI)) + return GI->getIntrinsicID(); + return Intrinsic::not_intrinsic; +} + template <> Printable MachineSSAContext::print(const MachineBasicBlock *Block) const { if (!Block) @@ -83,3 +91,8 @@ template <> Printable MachineSSAContext::print(Register Value) const { } }); } + +template <> +Printable MachineSSAContext::printAsOperand(const MachineBasicBlock *BB) const { + return Printable([BB](raw_ostream &Out) { BB->printAsOperand(Out); }); +} diff --git a/llvm/lib/IR/CMakeLists.txt b/llvm/lib/IR/CMakeLists.txt index 217fe703dd4ee..d9656a24d0ed3 100644 --- a/llvm/lib/IR/CMakeLists.txt +++ b/llvm/lib/IR/CMakeLists.txt @@ -10,6 +10,7 @@ add_llvm_component_library(LLVMCore ConstantFold.cpp ConstantRange.cpp Constants.cpp + ConvergenceVerifier.cpp Core.cpp CycleInfo.cpp DIBuilder.cpp diff --git a/llvm/lib/IR/ConvergenceVerifier.cpp b/llvm/lib/IR/ConvergenceVerifier.cpp new file mode 100644 index 0000000000000..e1c15a3a84083 --- /dev/null +++ b/llvm/lib/IR/ConvergenceVerifier.cpp @@ -0,0 +1,83 @@ +//===- ConvergenceVerifier.cpp - Verify convergence control -----*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// NOTE: Including the following header causes a premature instantiation of the +// template, and the compiler complains about explicit specialization after +// instantiation. So don't include it. +// +// #include "llvm/IR/ConvergenceVerifier.h" +//===----------------------------------------------------------------------===// + +#include "llvm/ADT/GenericConvergenceVerifierImpl.h" +#include "llvm/IR/Dominators.h" +#include "llvm/IR/Instructions.h" +#include "llvm/IR/SSAContext.h" + +using namespace llvm; + +template <> +const Instruction * +GenericConvergenceVerifier::findAndCheckConvergenceTokenUsed( + const Instruction &I) { + auto *CB = dyn_cast(&I); + if (!CB) + return nullptr; + + unsigned Count = + CB->countOperandBundlesOfType(LLVMContext::OB_convergencectrl); + CheckOrNull(Count <= 1, + "The 'convergencetrl' bundle can occur at most once on a call", + {Context.print(CB)}); + if (!Count) + return nullptr; + + auto Bundle = CB->getOperandBundle(LLVMContext::OB_convergencectrl); + CheckOrNull(Bundle->Inputs.size() == 1 && + Bundle->Inputs[0]->getType()->isTokenTy(), + "The 'convergencectrl' bundle requires exactly one token use.", + {Context.print(CB)}); + auto *Token = Bundle->Inputs[0].get(); + auto *Def = dyn_cast(Token); + + CheckOrNull( + Def && isConvergenceControlIntrinsic(SSAContext::getIntrinsicID(*Def)), + "Convergence control tokens can only be produced by calls to the " + "convergence control intrinsics.", + {Context.print(Token), Context.print(&I)}); + + if (Def) + Tokens[&I] = Def; + + return Def; +} + +template <> +bool GenericConvergenceVerifier::isConvergent( + const InstructionT &I) const { + if (auto *CB = dyn_cast(&I)) { + return CB->isConvergent(); + } + return false; +} + +template <> +bool GenericConvergenceVerifier::isControlledConvergent( + const InstructionT &I) { + // First find a token and place it in the map. + if (findAndCheckConvergenceTokenUsed(I)) + return true; + + // The entry and anchor intrinsics do not use a token, so we do a broad check + // here. The loop intrinsic will be checked separately for a missing token. + if (isConvergenceControlIntrinsic(SSAContext::getIntrinsicID(I))) + return true; + + return false; +} + +template class llvm::GenericConvergenceVerifier; diff --git a/llvm/lib/IR/SSAContext.cpp b/llvm/lib/IR/SSAContext.cpp index 3a5c4bf4aa30c..220abe3083ebd 100644 --- a/llvm/lib/IR/SSAContext.cpp +++ b/llvm/lib/IR/SSAContext.cpp @@ -16,10 +16,10 @@ #include "llvm/IR/Argument.h" #include "llvm/IR/BasicBlock.h" #include "llvm/IR/Function.h" -#include "llvm/IR/Instruction.h" #include "llvm/IR/Instructions.h" -#include "llvm/Support/raw_ostream.h" +#include "llvm/IR/Intrinsics.h" #include "llvm/IR/ModuleSlotTracker.h" +#include "llvm/Support/raw_ostream.h" using namespace llvm; @@ -69,6 +69,12 @@ bool SSAContext::isConstantOrUndefValuePhi(const Instruction &Instr) { return false; } +template <> Intrinsic::ID SSAContext::getIntrinsicID(const Instruction &I) { + if (auto *CB = dyn_cast(&I)) + return CB->getIntrinsicID(); + return Intrinsic::not_intrinsic; +} + template <> Printable SSAContext::print(const Value *V) const { return Printable([V](raw_ostream &Out) { V->print(Out); }); } @@ -89,3 +95,7 @@ template <> Printable SSAContext::print(const BasicBlock *BB) const { Out << MST.getLocalSlot(BB); }); } + +template <> Printable SSAContext::printAsOperand(const BasicBlock *BB) const { + return Printable([BB](raw_ostream &Out) { BB->printAsOperand(Out); }); +} diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp index 21acb4185f7a8..e03e6a117a243 100644 --- a/llvm/lib/IR/Verifier.cpp +++ b/llvm/lib/IR/Verifier.cpp @@ -73,7 +73,7 @@ #include "llvm/IR/Constant.h" #include "llvm/IR/ConstantRange.h" #include "llvm/IR/Constants.h" -#include "llvm/IR/CycleInfo.h" +#include "llvm/IR/ConvergenceVerifier.h" #include "llvm/IR/DataLayout.h" #include "llvm/IR/DebugInfo.h" #include "llvm/IR/DebugInfoMetadata.h" @@ -329,13 +329,6 @@ class Verifier : public InstVisitor, VerifierSupport { /// The current source language. dwarf::SourceLanguage CurrentSourceLang = dwarf::DW_LANG_lo_user; - /// Whether the current function has convergencectrl operand bundles. - enum { - ControlledConvergence, - UncontrolledConvergence, - NoConvergence - } ConvergenceKind = NoConvergence; - /// Whether source was present on the first DIFile encountered in each CU. DenseMap HasSourceDebugInfo; @@ -370,6 +363,7 @@ class Verifier : public InstVisitor, VerifierSupport { SmallVector DebugFnArgs; TBAAVerifier TBAAVerifyHelper; + ConvergenceVerifier CV; SmallVector NoAliasScopeDecls; @@ -411,12 +405,19 @@ class Verifier : public InstVisitor, VerifierSupport { return false; } + auto FailureCB = [this](const Twine &Message) { + this->CheckFailed(Message); + }; + CV.initialize(*OS, FailureCB, F); + Broken = false; // FIXME: We strip const here because the inst visitor strips const. visit(const_cast(F)); verifySiblingFuncletUnwinds(); - if (ConvergenceKind == ControlledConvergence) - verifyConvergenceControl(const_cast(F)); + + if (CV.sawTokens()) + CV.verify(DT); + InstsInThisBlock.clear(); DebugFnArgs.clear(); LandingPadResultTy = nullptr; @@ -424,7 +425,6 @@ class Verifier : public InstVisitor, VerifierSupport { SiblingFuncletInfo.clear(); verifyNoAliasScopeDecl(); NoAliasScopeDecls.clear(); - ConvergenceKind = NoConvergence; return !Broken; } @@ -600,7 +600,6 @@ class Verifier : public InstVisitor, VerifierSupport { void verifyStatepoint(const CallBase &Call); void verifyFrameRecoverIndices(); void verifySiblingFuncletUnwinds(); - void verifyConvergenceControl(Function &F); void verifyFragmentExpression(const DbgVariableIntrinsic &I); template @@ -2535,138 +2534,6 @@ void Verifier::verifySiblingFuncletUnwinds() { } } -static bool isConvergenceControlIntrinsic(const CallBase &Call) { - switch (Call.getIntrinsicID()) { - case Intrinsic::experimental_convergence_anchor: - case Intrinsic::experimental_convergence_entry: - case Intrinsic::experimental_convergence_loop: - return true; - default: - return false; - } -} - -static bool isControlledConvergent(const CallBase &Call) { - if (Call.countOperandBundlesOfType(LLVMContext::OB_convergencectrl)) - return true; - return isConvergenceControlIntrinsic(Call); -} - -void Verifier::verifyConvergenceControl(Function &F) { - DenseMap> LiveTokenMap; - DenseMap CycleHearts; - - // Just like the DominatorTree, compute the CycleInfo locally so that we - // can run the verifier outside of a pass manager and we don't rely on - // potentially out-dated analysis results. - CycleInfo CI; - CI.compute(F); - - auto checkBundle = [&](OperandBundleUse &Bundle, CallBase *CB, - SmallVectorImpl &LiveTokens) { - Check(Bundle.Inputs.size() == 1 && Bundle.Inputs[0]->getType()->isTokenTy(), - "The 'convergencectrl' bundle requires exactly one token use.", CB); - - Value *Token = Bundle.Inputs[0].get(); - auto *Def = dyn_cast(Token); - Check(Def && isConvergenceControlIntrinsic(*Def), - "Convergence control tokens can only be produced by calls to the " - "convergence control intrinsics.", - Token, CB); - - Check(llvm::is_contained(LiveTokens, Token), - "Convergence region is not well-nested.", Token, CB); - - while (LiveTokens.back() != Token) - LiveTokens.pop_back(); - - // Check static rules about cycles. - auto *BB = CB->getParent(); - auto *BBCycle = CI.getCycle(BB); - if (!BBCycle) - return; - - BasicBlock *DefBB = Def->getParent(); - if (DefBB == BB || BBCycle->contains(DefBB)) { - // degenerate occurrence of a loop intrinsic - return; - } - - auto *II = dyn_cast(CB); - Check(II && - II->getIntrinsicID() == Intrinsic::experimental_convergence_loop, - "Convergence token used by an instruction other than " - "llvm.experimental.convergence.loop in a cycle that does " - "not contain the token's definition.", - CB, CI.print(BBCycle)); - - while (true) { - auto *Parent = BBCycle->getParentCycle(); - if (!Parent || Parent->contains(DefBB)) - break; - BBCycle = Parent; - }; - - Check(BBCycle->isReducible() && BB == BBCycle->getHeader(), - "Cycle heart must dominate all blocks in the cycle.", CB, BB, - CI.print(BBCycle)); - Check(!CycleHearts.count(BBCycle), - "Two static convergence token uses in a cycle that does " - "not contain either token's definition.", - CB, CycleHearts[BBCycle], CI.print(BBCycle)); - CycleHearts[BBCycle] = CB; - }; - - ReversePostOrderTraversal RPOT(&F); - SmallVector LiveTokens; - for (BasicBlock *BB : RPOT) { - LiveTokens.clear(); - auto LTIt = LiveTokenMap.find(BB); - if (LTIt != LiveTokenMap.end()) { - LiveTokens = std::move(LTIt->second); - LiveTokenMap.erase(LTIt); - } - - for (Instruction &I : *BB) { - CallBase *CB = dyn_cast(&I); - if (!CB) - continue; - - Check(CB->countOperandBundlesOfType(LLVMContext::OB_convergencectrl) <= 1, - "The 'convergencetrl' bundle can occur at most once on a call", CB); - - auto Bundle = CB->getOperandBundle(LLVMContext::OB_convergencectrl); - if (Bundle) - checkBundle(*Bundle, CB, LiveTokens); - - if (CB->getType()->isTokenTy()) - LiveTokens.push_back(CB); - } - - // Propagate token liveness - for (BasicBlock *Succ : successors(BB)) { - DomTreeNode *SuccNode = DT.getNode(Succ); - LTIt = LiveTokenMap.find(Succ); - if (LTIt == LiveTokenMap.end()) { - // We're the first predecessor: all tokens which dominate the - // successor are live for now. - LTIt = LiveTokenMap.try_emplace(Succ).first; - for (CallBase *LiveToken : LiveTokens) { - if (!DT.dominates(DT.getNode(LiveToken->getParent()), SuccNode)) - break; - LTIt->second.push_back(LiveToken); - } - } else { - // Compute the intersection of live tokens. - auto It = llvm::partition(LTIt->second, [&LiveTokens](CallBase *Token) { - return llvm::is_contained(LiveTokens, Token); - }); - LTIt->second.erase(It, LTIt->second.end()); - } - } - } -} - // visitFunction - Verify that a function is ok. // void Verifier::visitFunction(const Function &F) { @@ -3688,22 +3555,7 @@ void Verifier::visitCallBase(CallBase &Call) { if (Call.isInlineAsm()) verifyInlineAsmCall(Call); - if (isControlledConvergent(Call)) { - Check(Call.isConvergent(), - "Expected convergent attribute on a controlled convergent call.", - Call); - Check(ConvergenceKind != UncontrolledConvergence, - "Cannot mix controlled and uncontrolled convergence in the same " - "function.", - Call); - ConvergenceKind = ControlledConvergence; - } else if (Call.isConvergent()) { - Check(ConvergenceKind != ControlledConvergence, - "Cannot mix controlled and uncontrolled convergence in the same " - "function.", - Call); - ConvergenceKind = UncontrolledConvergence; - } + CV.visit(Call); visitInstruction(Call); }