diff --git a/llvm/include/llvm/Analysis/MemorySSA.h b/llvm/include/llvm/Analysis/MemorySSA.h index 4af95873332601..2b91353bce7b16 100644 --- a/llvm/include/llvm/Analysis/MemorySSA.h +++ b/llvm/include/llvm/Analysis/MemorySSA.h @@ -786,7 +786,7 @@ class MemorySSA { /// Used in various insertion functions to specify whether we are talking /// about the beginning or end of a block. - enum InsertionPlace { Beginning, End }; + enum InsertionPlace { Beginning, End, BeforeTerminator }; protected: // Used by Memory SSA annotater, dumpers, and wrapper pass diff --git a/llvm/lib/Analysis/LoopInfo.cpp b/llvm/lib/Analysis/LoopInfo.cpp index 663b42c55f1ec0..3e3303cbb02397 100644 --- a/llvm/lib/Analysis/LoopInfo.cpp +++ b/llvm/lib/Analysis/LoopInfo.cpp @@ -106,7 +106,8 @@ bool Loop::makeLoopInvariant(Instruction *I, bool &Changed, I->moveBefore(InsertPt); if (MSSAU) if (auto *MUD = MSSAU->getMemorySSA()->getMemoryAccess(I)) - MSSAU->moveToPlace(MUD, InsertPt->getParent(), MemorySSA::End); + MSSAU->moveToPlace(MUD, InsertPt->getParent(), + MemorySSA::BeforeTerminator); // There is possibility of hoisting this instruction above some arbitrary // condition. Any metadata defined on it can be control dependent on this diff --git a/llvm/lib/Analysis/MemorySSAUpdater.cpp b/llvm/lib/Analysis/MemorySSAUpdater.cpp index f2d56b05d968ed..473268982f2df9 100644 --- a/llvm/lib/Analysis/MemorySSAUpdater.cpp +++ b/llvm/lib/Analysis/MemorySSAUpdater.cpp @@ -1159,7 +1159,13 @@ void MemorySSAUpdater::moveAfter(MemoryUseOrDef *What, MemoryUseOrDef *Where) { void MemorySSAUpdater::moveToPlace(MemoryUseOrDef *What, BasicBlock *BB, MemorySSA::InsertionPlace Where) { - return moveTo(What, BB, Where); + if (Where != MemorySSA::InsertionPlace::BeforeTerminator) + return moveTo(What, BB, Where); + + if (auto *Where = MSSA->getMemoryAccess(BB->getTerminator())) + return moveBefore(What, Where); + else + return moveTo(What, BB, MemorySSA::InsertionPlace::End); } // All accesses in To used to be in From. Move to end and update access lists. diff --git a/llvm/lib/Transforms/Scalar/GVNHoist.cpp b/llvm/lib/Transforms/Scalar/GVNHoist.cpp index bc85c1b62cf2b0..e1796f6bf05a8a 100644 --- a/llvm/lib/Transforms/Scalar/GVNHoist.cpp +++ b/llvm/lib/Transforms/Scalar/GVNHoist.cpp @@ -957,7 +957,8 @@ class GVNHoist { if (MoveAccess && NewMemAcc) { // The definition of this ld/st will not change: ld/st hoisting is // legal when the ld/st is not moved past its current definition. - MSSAUpdater->moveToPlace(NewMemAcc, DestBB, MemorySSA::End); + MSSAUpdater->moveToPlace(NewMemAcc, DestBB, + MemorySSA::BeforeTerminator); } // Replace all other instructions with Repl with memory access NewMemAcc. @@ -1068,6 +1069,9 @@ class GVNHoist { ++NI; } + if (MSSA && VerifyMemorySSA) + MSSA->verifyMemorySSA(); + NumHoisted += NL + NS + NC + NI; NumRemoved += NR; NumLoadsHoisted += NL; diff --git a/llvm/lib/Transforms/Scalar/LICM.cpp b/llvm/lib/Transforms/Scalar/LICM.cpp index 4927ee314a5e75..8c33045c238027 100644 --- a/llvm/lib/Transforms/Scalar/LICM.cpp +++ b/llvm/lib/Transforms/Scalar/LICM.cpp @@ -1501,7 +1501,8 @@ static void moveInstructionBefore(Instruction &I, Instruction &Dest, if (MSSAU) if (MemoryUseOrDef *OldMemAcc = cast_or_null( MSSAU->getMemorySSA()->getMemoryAccess(&I))) - MSSAU->moveToPlace(OldMemAcc, Dest.getParent(), MemorySSA::End); + MSSAU->moveToPlace(OldMemAcc, Dest.getParent(), + MemorySSA::BeforeTerminator); if (SE) SE->forgetValue(&I); } diff --git a/llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp b/llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp index 95b015a3e41379..13e44765985f47 100644 --- a/llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp +++ b/llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp @@ -2428,7 +2428,7 @@ turnGuardIntoBranch(IntrinsicInst *GI, Loop &L, if (MSSAU) { MemoryDef *MD = cast(MSSAU->getMemorySSA()->getMemoryAccess(GI)); - MSSAU->moveToPlace(MD, DeoptBlock, MemorySSA::End); + MSSAU->moveToPlace(MD, DeoptBlock, MemorySSA::BeforeTerminator); if (VerifyMemorySSA) MSSAU->getMemorySSA()->verifyMemorySSA(); } diff --git a/llvm/test/Analysis/MemorySSA/pr44027.ll b/llvm/test/Analysis/MemorySSA/pr44027.ll new file mode 100644 index 00000000000000..3c0f9266ca2306 --- /dev/null +++ b/llvm/test/Analysis/MemorySSA/pr44027.ll @@ -0,0 +1,27 @@ +; RUN: opt -gvn-hoist -verify-memoryssa -S < %s | FileCheck %s +; REQUIRES: asserts + +target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128" +target triple = "x86_64-unknown-linux-gnu" + +declare i32 @wobble(...) + +declare void @spam() align 2 + +; CHECK-LABEL: @f() +define void @f() personality i8* bitcast (i32 (...)* @wobble to i8*) { +bb: + %tmp = alloca i32*, align 8 + invoke void @spam() + to label %bb16 unwind label %bb23 + +bb16: ; preds = %bb + %tmp17 = load i32*, i32** %tmp, align 8 + %tmp18 = load i32*, i32** %tmp, align 8 + unreachable + +bb23: ; preds = %bb + %tmp24 = landingpad { i8*, i32 } + cleanup + unreachable +}