diff --git a/llvm/include/llvm/IR/IntrinsicInst.h b/llvm/include/llvm/IR/IntrinsicInst.h index 12c464f8b8babf..9d68f3fdde6c24 100644 --- a/llvm/include/llvm/IR/IntrinsicInst.h +++ b/llvm/include/llvm/IR/IntrinsicInst.h @@ -1005,6 +1005,11 @@ class NoAliasScopeDeclInst : public IntrinsicInst { cast(getOperand(Intrinsic::NoAliasScopeDeclScopeArg)); return cast(MV->getMetadata()); } + + void setScopeList(MDNode *ScopeList) { + setOperand(Intrinsic::NoAliasScopeDeclScopeArg, + MetadataAsValue::get(getContext(), ScopeList)); + } }; } // end namespace llvm diff --git a/llvm/include/llvm/Transforms/Utils/Cloning.h b/llvm/include/llvm/Transforms/Utils/Cloning.h index 16062fb2f5f54b..56aaa5d48e2a1f 100644 --- a/llvm/include/llvm/Transforms/Utils/Cloning.h +++ b/llvm/include/llvm/Transforms/Utils/Cloning.h @@ -272,20 +272,16 @@ void updateProfileCallee( /// basic blocks and extract their scope. These are candidates for duplication /// when cloning. void identifyNoAliasScopesToClone( - ArrayRef BBs, - SmallVectorImpl &NoAliasDeclScopes); + ArrayRef BBs, SmallVectorImpl &NoAliasDeclScopes); /// Duplicate the specified list of noalias decl scopes. /// The 'Ext' string is added as an extension to the name. -/// Afterwards, the ClonedMVScopes contains a mapping of the original MV onto -/// the cloned version. -/// The ClonedScopes contains the mapping of the original scope MDNode onto the -/// cloned scope. +/// Afterwards, the ClonedScopes contains the mapping of the original scope +/// MDNode onto the cloned scope. /// Be aware that the cloned scopes are still part of the original scope domain. void cloneNoAliasScopes( - ArrayRef NoAliasDeclScopes, + ArrayRef NoAliasDeclScopes, DenseMap &ClonedScopes, - DenseMap &ClonedMVScopes, StringRef Ext, LLVMContext &Context); /// Adapt the metadata for the specified instruction according to the @@ -293,20 +289,19 @@ void cloneNoAliasScopes( /// some noalias scopes needed to be cloned. void adaptNoAliasScopes( llvm::Instruction *I, const DenseMap &ClonedScopes, - const DenseMap &ClonedMVScopes, LLVMContext &Context); /// Clone the specified noalias decl scopes. Then adapt all instructions in the /// NewBlocks basicblocks to the cloned versions. /// 'Ext' will be added to the duplicate scope names. -void cloneAndAdaptNoAliasScopes(ArrayRef NoAliasDeclScopes, +void cloneAndAdaptNoAliasScopes(ArrayRef NoAliasDeclScopes, ArrayRef NewBlocks, LLVMContext &Context, StringRef Ext); /// Clone the specified noalias decl scopes. Then adapt all instructions in the /// [IStart, IEnd] (IEnd included !) range to the cloned versions. 'Ext' will be /// added to the duplicate scope names. -void cloneAndAdaptNoAliasScopes(ArrayRef NoAliasDeclScopes, +void cloneAndAdaptNoAliasScopes(ArrayRef NoAliasDeclScopes, Instruction *IStart, Instruction *IEnd, LLVMContext &Context, StringRef Ext); } // end namespace llvm diff --git a/llvm/lib/Transforms/Utils/CloneFunction.cpp b/llvm/lib/Transforms/Utils/CloneFunction.cpp index ac474fbac7b380..51a49574e55d81 100644 --- a/llvm/lib/Transforms/Utils/CloneFunction.cpp +++ b/llvm/lib/Transforms/Utils/CloneFunction.cpp @@ -886,15 +886,13 @@ BasicBlock *llvm::DuplicateInstructionsInSplitBetween( } void llvm::cloneNoAliasScopes( - ArrayRef NoAliasDeclScopes, + ArrayRef NoAliasDeclScopes, DenseMap &ClonedScopes, - DenseMap &ClonedMVScopes, StringRef Ext, LLVMContext &Context) { MDBuilder MDB(Context); - for (auto *MV : NoAliasDeclScopes) { - SmallVector ScopeList; - for (auto &MDOperand : cast(MV->getMetadata())->operands()) { + for (auto *ScopeList : NoAliasDeclScopes) { + for (auto &MDOperand : ScopeList->operands()) { if (MDNode *MD = dyn_cast(MDOperand)) { AliasScopeNode SNANode(MD); @@ -908,94 +906,86 @@ void llvm::cloneNoAliasScopes( MDNode *NewScope = MDB.createAnonymousAliasScope( const_cast(SNANode.getDomain()), Name); ClonedScopes.insert(std::make_pair(MD, NewScope)); - ScopeList.push_back(NewScope); } } - MDNode *NewScopeList = MDNode::get(Context, ScopeList); - ClonedMVScopes.insert( - std::make_pair(MV, MetadataAsValue::get(Context, NewScopeList))); } } void llvm::adaptNoAliasScopes( Instruction *I, const DenseMap &ClonedScopes, - const DenseMap &ClonedMVScopes, LLVMContext &Context) { - // MetadataAsValue will always be replaced ! - for (Use &U : I->operands()) - if (MetadataAsValue *MV = dyn_cast(U)) - if (auto *NewMV = ClonedMVScopes.lookup(MV)) - U.set(NewMV); - - auto replaceWhenNeeded = [&](unsigned MD_ID) { - if (const MDNode *CSNoAlias = I->getMetadata(MD_ID)) { - bool NeedsReplacement = false; - SmallVector NewScopeList; - for (auto &MDOp : CSNoAlias->operands()) { - if (MDNode *MD = dyn_cast(MDOp)) { - if (auto *NewMD = ClonedScopes.lookup(MD)) { - NewScopeList.push_back(NewMD); - NeedsReplacement = true; - continue; - } - NewScopeList.push_back(MD); + auto CloneScopeList = [&](const MDNode *ScopeList) -> MDNode * { + bool NeedsReplacement = false; + SmallVector NewScopeList; + for (auto &MDOp : ScopeList->operands()) { + if (MDNode *MD = dyn_cast(MDOp)) { + if (auto *NewMD = ClonedScopes.lookup(MD)) { + NewScopeList.push_back(NewMD); + NeedsReplacement = true; + continue; } + NewScopeList.push_back(MD); } - if (NeedsReplacement) - I->setMetadata(MD_ID, MDNode::get(Context, NewScopeList)); } + if (NeedsReplacement) + return MDNode::get(Context, NewScopeList); + return nullptr; + }; + + if (auto *Decl = dyn_cast(I)) + if (auto *NewScopeList = CloneScopeList(Decl->getScopeList())) + Decl->setScopeList(NewScopeList); + + auto replaceWhenNeeded = [&](unsigned MD_ID) { + if (const MDNode *CSNoAlias = I->getMetadata(MD_ID)) + if (auto *NewScopeList = CloneScopeList(CSNoAlias)) + I->setMetadata(MD_ID, NewScopeList); }; replaceWhenNeeded(LLVMContext::MD_noalias); replaceWhenNeeded(LLVMContext::MD_alias_scope); } void llvm::cloneAndAdaptNoAliasScopes( - ArrayRef NoAliasDeclScopes, + ArrayRef NoAliasDeclScopes, ArrayRef NewBlocks, LLVMContext &Context, StringRef Ext) { if (NoAliasDeclScopes.empty()) return; DenseMap ClonedScopes; - DenseMap ClonedMVScopes; LLVM_DEBUG(dbgs() << "cloneAndAdaptNoAliasScopes: cloning " << NoAliasDeclScopes.size() << " node(s)\n"); - cloneNoAliasScopes(NoAliasDeclScopes, ClonedScopes, ClonedMVScopes, Ext, - Context); + cloneNoAliasScopes(NoAliasDeclScopes, ClonedScopes, Ext, Context); // Identify instructions using metadata that needs adaptation for (BasicBlock *NewBlock : NewBlocks) for (Instruction &I : *NewBlock) - adaptNoAliasScopes(&I, ClonedScopes, ClonedMVScopes, Context); + adaptNoAliasScopes(&I, ClonedScopes, Context); } void llvm::cloneAndAdaptNoAliasScopes( - ArrayRef NoAliasDeclScopes, Instruction *IStart, + ArrayRef NoAliasDeclScopes, Instruction *IStart, Instruction *IEnd, LLVMContext &Context, StringRef Ext) { if (NoAliasDeclScopes.empty()) return; DenseMap ClonedScopes; - DenseMap ClonedMVScopes; LLVM_DEBUG(dbgs() << "cloneAndAdaptNoAliasScopes: cloning " << NoAliasDeclScopes.size() << " node(s)\n"); - cloneNoAliasScopes(NoAliasDeclScopes, ClonedScopes, ClonedMVScopes, Ext, - Context); + cloneNoAliasScopes(NoAliasDeclScopes, ClonedScopes, Ext, Context); // Identify instructions using metadata that needs adaptation assert(IStart->getParent() == IEnd->getParent() && "different basic block ?"); auto ItStart = IStart->getIterator(); auto ItEnd = IEnd->getIterator(); ++ItEnd; // IEnd is included, increment ItEnd to get the end of the range for (auto &I : llvm::make_range(ItStart, ItEnd)) - adaptNoAliasScopes(&I, ClonedScopes, ClonedMVScopes, Context); + adaptNoAliasScopes(&I, ClonedScopes, Context); } void llvm::identifyNoAliasScopesToClone( - ArrayRef BBs, - SmallVectorImpl &NoAliasDeclScopes) { + ArrayRef BBs, SmallVectorImpl &NoAliasDeclScopes) { for (BasicBlock *BB : BBs) for (Instruction &I : *BB) if (auto *Decl = dyn_cast(&I)) - NoAliasDeclScopes.push_back(cast( - Decl->getOperand(Intrinsic::NoAliasScopeDeclScopeArg))); + NoAliasDeclScopes.push_back(Decl->getScopeList()); } diff --git a/llvm/lib/Transforms/Utils/InlineFunction.cpp b/llvm/lib/Transforms/Utils/InlineFunction.cpp index 2f8f3666d39643..0ac8fa537f4e34 100644 --- a/llvm/lib/Transforms/Utils/InlineFunction.cpp +++ b/llvm/lib/Transforms/Utils/InlineFunction.cpp @@ -927,11 +927,8 @@ void ScopedAliasMetadataDeepCloner::remap(ValueToValueMapTy &VMap) { if (MDNode *M = I->getMetadata(LLVMContext::MD_noalias)) I->setMetadata(LLVMContext::MD_noalias, MDMap[M]); - if (auto *Decl = dyn_cast(I)) { - auto *NewMV = - MetadataAsValue::get(Decl->getContext(), MDMap[Decl->getScopeList()]); - Decl->setOperand(Intrinsic::NoAliasScopeDeclScopeArg, NewMV); - } + if (auto *Decl = dyn_cast(I)) + Decl->setScopeList(MDMap[Decl->getScopeList()]); } } diff --git a/llvm/lib/Transforms/Utils/LoopRotationUtils.cpp b/llvm/lib/Transforms/Utils/LoopRotationUtils.cpp index 8192092822aa41..b678efdc8d88dc 100644 --- a/llvm/lib/Transforms/Utils/LoopRotationUtils.cpp +++ b/llvm/lib/Transforms/Utils/LoopRotationUtils.cpp @@ -404,7 +404,7 @@ bool LoopRotate::rotateLoop(Loop *L, bool SimplifiedLatch) { // Remember the local noalias scope declarations in the header. After the // rotation, they must be duplicated and the scope must be cloned. This // avoids unwanted interaction across iterations. - SmallVector NoAliasDeclInstructions; + SmallVector NoAliasDeclInstructions; for (Instruction &I : *OrigHeader) if (auto *Decl = dyn_cast(&I)) NoAliasDeclInstructions.push_back(Decl); @@ -493,7 +493,7 @@ bool LoopRotate::rotateLoop(Loop *L, bool SimplifiedLatch) { // Clone the llvm.experimental.noalias.decl again for the NewHeader. Instruction *NewHeaderInsertionPoint = &(*NewHeader->getFirstNonPHI()); - for (Instruction *NAD : NoAliasDeclInstructions) { + for (NoAliasScopeDeclInst *NAD : NoAliasDeclInstructions) { LLVM_DEBUG(dbgs() << " Cloning llvm.experimental.noalias.scope.decl:" << *NAD << "\n"); Instruction *NewNAD = NAD->clone(); @@ -505,10 +505,9 @@ bool LoopRotate::rotateLoop(Loop *L, bool SimplifiedLatch) { { auto &Context = NewHeader->getContext(); - SmallVector NoAliasDeclScopes; - for (Instruction *NAD : NoAliasDeclInstructions) - NoAliasDeclScopes.push_back(cast( - NAD->getOperand(Intrinsic::NoAliasScopeDeclScopeArg))); + SmallVector NoAliasDeclScopes; + for (NoAliasScopeDeclInst *NAD : NoAliasDeclInstructions) + NoAliasDeclScopes.push_back(NAD->getScopeList()); LLVM_DEBUG(dbgs() << " Updating OrigHeader scopes\n"); cloneAndAdaptNoAliasScopes(NoAliasDeclScopes, {OrigHeader}, Context, diff --git a/llvm/lib/Transforms/Utils/LoopUnroll.cpp b/llvm/lib/Transforms/Utils/LoopUnroll.cpp index ed1de9b1aee556..d4cd574052399a 100644 --- a/llvm/lib/Transforms/Utils/LoopUnroll.cpp +++ b/llvm/lib/Transforms/Utils/LoopUnroll.cpp @@ -592,7 +592,7 @@ LoopUnrollResult llvm::UnrollLoop(Loop *L, UnrollLoopOptions ULO, LoopInfo *LI, // Identify what noalias metadata is inside the loop: if it is inside the // loop, the associated metadata must be cloned for each iteration. - SmallVector LoopLocalNoAliasDeclScopes; + SmallVector LoopLocalNoAliasDeclScopes; identifyNoAliasScopesToClone(L->getBlocks(), LoopLocalNoAliasDeclScopes); for (unsigned It = 1; It != ULO.Count; ++It) {