Revert "[SLP] Support memory runtime alias checks"

[alexey-bataev]

This reverts commit d4c6c86 to address
postcommit review #203631

[llvmorg-github-actions]

@llvm/pr-subscribers-vectorizers

@llvm/pr-subscribers-llvm-transforms

Author: Alexey Bataev (alexey-bataev)

Changes

This reverts commit d4c6c86 to address
postcommit review #203631

---

Patch is 73.56 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/209173.diff

4 Files Affected:

• (modified) llvm/include/llvm/Transforms/Vectorize/SLPVectorizer.h (-13)
• (modified) llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp (+17-825)
• (modified) llvm/test/Transforms/SLPVectorizer/AArch64/loadi8.ll (+10-41)
• (modified) llvm/test/Transforms/SLPVectorizer/X86/runtime-alias-checks.ll (-315)

diff --git a/llvm/include/llvm/Transforms/Vectorize/SLPVectorizer.h b/llvm/include/llvm/Transforms/Vectorize/SLPVectorizer.h
index 538d62626b37b..b3f5718ddfda4 100644
--- a/llvm/include/llvm/Transforms/Vectorize/SLPVectorizer.h
+++ b/llvm/include/llvm/Transforms/Vectorize/SLPVectorizer.h
@@ -172,25 +172,12 @@ struct SLPVectorizerPass : public OptionalPassInfoMixin<SLPVectorizerPass> {
                                           unsigned Idx, unsigned MinVF,
                                           unsigned &Size);
 
-  /// Single vectorization attempt for a store chain. \p vectorizeStoreChain
-  /// wraps this to retry once with runtime alias checks enabled when the
-  /// normal attempt is blocked only by runtime-checkable may-alias
-  /// dependencies.
-  std::optional<bool> vectorizeStoreChainImpl(ArrayRef<Value *> Chain,
-                                              slpvectorizer::BoUpSLP &R,
-                                              unsigned Idx, unsigned MinVF,
-                                              unsigned &Size);
-
   bool vectorizeStores(
       ArrayRef<StoreInst *> Stores, slpvectorizer::BoUpSLP &R,
       DenseSet<std::tuple<Value *, Value *, Value *, Value *, unsigned>>
           &Visited,
       bool AllowMaskedStores = true);
 
-  /// Set by runImpl() when runtime alias check versioning changed the CFG, so
-  /// run() can drop CFG-analysis preservation only when necessary.
-  bool CFGChanged = false;
-
   /// The store instructions in a basic block organized by base pointer.
   StoreListMap Stores;
 
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index a70aa1d35e02f..7e22ba3bd149c 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -38,7 +38,6 @@
 #include "llvm/Analysis/CodeMetrics.h"
 #include "llvm/Analysis/ConstantFolding.h"
 #include "llvm/Analysis/DemandedBits.h"
-#include "llvm/Analysis/DomTreeUpdater.h"
 #include "llvm/Analysis/GlobalsModRef.h"
 #include "llvm/Analysis/IVDescriptors.h"
 #include "llvm/Analysis/Loads.h"
@@ -53,7 +52,6 @@
 #include "llvm/Analysis/VectorUtils.h"
 #include "llvm/IR/Attributes.h"
 #include "llvm/IR/BasicBlock.h"
-#include "llvm/IR/CFG.h"
 #include "llvm/IR/Constant.h"
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/DataLayout.h"
@@ -291,28 +289,6 @@ static cl::opt<bool> PerLaneGatherScale(
     cl::desc("Use per-lane execution scale for gather/buildvector tree "
              "entries to model LICM-hoistable buildvector sequences."));
 
-/// Enable versioning of a basic block with runtime alias checks.
-static cl::opt<bool> SLPEnableRuntimeAliasChecks(
-    "slp-vectorize-with-runtime-alias-checks", cl::init(true), cl::Hidden,
-    cl::desc("Allow SLP to version a block with runtime alias checks to "
-             "vectorize trees blocked by may-alias memory dependencies."));
-
-/// Maximum number of runtime alias checks (one per pair of base objects) that
-/// may guard a single versioned region.
-static cl::opt<unsigned> SLPMaxRuntimeAliasChecks(
-    "slp-max-runtime-alias-checks", cl::init(8), cl::Hidden,
-    cl::desc("The maximum number of runtime alias checks generated to guard a "
-             "single SLP-vectorized region."));
-
-/// The runtime checks and the guard branch execute on both the vector and the
-/// scalar fallback path, so they add overhead to the scalar code.
-static cl::opt<unsigned> SLPRuntimeAliasChecksMaxScalarCostPercent(
-    "slp-runtime-alias-checks-max-scalar-cost-percent", cl::init(25),
-    cl::Hidden,
-    cl::desc("Maximum SLP runtime alias check cost, as a percentage of the "
-             "guarded scalar region cost, before versioning is rejected to "
-             "avoid pessimizing the scalar fallback path."));
-
 // Limit the number of alias checks. The limit is chosen so that
 // it has no negative effect on the llvm benchmarks.
 static const unsigned AliasedCheckLimit = 10;
@@ -2103,93 +2079,7 @@ class slpvectorizer::BoUpSLP {
   /// Construct a vectorizable tree that starts at \p Roots.
   void buildTree(ArrayRef<Value *> Roots);
 
-  /// Returns true if the last buildTree() observed a may-alias memory
-  /// dependency between two distinct, range-checkable base objects, i.e. a
-  /// dependency that could be turned into a runtime alias check.
-  bool hasRuntimeCheckableBlockers() const {
-    return HasRuntimeCheckableBlockers;
-  }
-
-  /// Records whether a may-alias dependency between distinct, range-checkable
-  /// base objects has been observed, so the caller can decide to retry with
-  /// runtime alias checks enabled.
-  void setHasRuntimeCheckableBlockers(bool V) {
-    HasRuntimeCheckableBlockers = V;
-  }
-
-  /// Returns true if the last buildTree() kept a may-alias memory dependency
-  /// that is not runtime-checkable (call or a non-simple mem access). Such a
-  /// dependency cannot be dropped, so a runtime-checks retry cannot unblock the
-  /// region and would be pure overhead.
-  bool hasNonCheckableMemBlocker() const { return HasNonCheckableMemBlocker; }
-
-  /// Records that a non-runtime-checkable may-alias dependency was kept.
-  void setHasNonCheckableMemBlocker(bool V) { HasNonCheckableMemBlocker = V; }
-
-  /// Returns true if the current vectorization attempt may drop
-  /// runtime-checkable may-alias dependencies and guard the region with
-  /// runtime alias checks.
-  bool isTryingRuntimeAliasChecks() const { return TryRuntimeAliasChecks; }
-
-  /// Enables or disables dropping runtime-checkable may-alias dependencies in
-  /// favor of runtime alias checks for the current vectorization attempt.
-  void setTryRuntimeAliasChecks(bool V) { TryRuntimeAliasChecks = V; }
-
-  /// Resets the runtime alias check data.
-  void resetRuntimeAliasCheckState() {
-    HasRuntimeCheckableBlockers = false;
-    HasNonCheckableMemBlocker = false;
-    RTChecksFinalized = false;
-    RTChecks.clear();
-    RTOrigBodyOrder.clear();
-  }
-
-  /// Snapshots RTChecks.BB's body (non-PHI, non-terminator) into
-  /// RTOrigBodyOrder in program order, for the scalar fallback.
-  void captureRuntimeCheckBodySnapshot();
-
-  /// Returns true if \p BB satisfies the block-level preconditions for runtime
-  /// alias check versioning (straight-line, outside any loop, duplicable, not a
-  /// scalar fallback, function not optimized for size). These checks do not
-  /// depend on the collected checks, so they can gate the (expensive)
-  /// optimistic retry before any tree is rebuilt.
-  bool canVersionBlockForRuntimeChecks(BasicBlock *BB) const;
-
-  /// Returns true if the runtime alias checks can be safely emitted to guard
-  /// the vectorized region.
-  bool canVersionForRuntimeChecks();
-
-  /// Returns true if \p BB is a scalar fallback block created by runtime alias
-  /// check versioning.
-  bool isScalarFallbackBlock(BasicBlock *BB) const {
-    return ScalarFallbackBlocks.contains(BB);
-  }
-
-  /// Returns true if an optimistic runtime-checks versioning attempt already
-  /// failed for \p BB, so further retries in the same block can be skipped.
-  bool runtimeChecksFailedForBlock(BasicBlock *BB) const {
-    return FailedRuntimeChecksBlocks.contains(BB);
-  }
-
-  /// Records that an optimistic runtime-checks versioning attempt failed for
-  /// \p BB.
-  void markRuntimeChecksFailedForBlock(BasicBlock *BB) {
-    FailedRuntimeChecksBlocks.insert(BB);
-  }
-
-  /// Returns the modeled cost of the runtime alias checks collected during the
-  /// last (optimistic) buildTree().
-  InstructionCost getRuntimeChecksCost() const;
-
-  /// Returns true if the last (optimistic) buildTree() collected any runtime
-  /// alias checks that must guard the vectorized region.
-  bool hasRuntimeAliasChecks() const { return !RTChecks.BasePairs.empty(); }
-
-  /// Returns true if vectorization changed the CFG (i.e. a block was versioned
-  /// with runtime alias checks). When true, CFG analyses must not be preserved.
-  bool isCFGChanged() const { return CFGChanged; }
-
-  /// Returns the scalars of the root node.
+  /// Return the scalars of the root node.
   ArrayRef<Value *> getRootNodeScalars() const {
     assert(!VectorizableTree.empty() && "No graph to get the first node from");
     return VectorizableTree.front()->Scalars;
@@ -2296,10 +2186,6 @@ class slpvectorizer::BoUpSLP {
     ExternalUses.clear();
     ExternalUsesAsOriginalScalar.clear();
     ExternalUsesWithNonUsers.clear();
-    RTChecks.clear();
-    HasRuntimeCheckableBlockers = false;
-    HasNonCheckableMemBlocker = false;
-    RTChecksFinalized = false;
     for (auto &Iter : BlocksSchedules) {
       BlockScheduling *BS = Iter.second.get();
       BS->clear();
@@ -5048,89 +4934,6 @@ class slpvectorizer::BoUpSLP {
     return Aliased;
   }
 
-  /// Returns true if the may-alias dependency between simple load/store
-  /// instructions \p Inst1 and \p Inst2 could be disambiguated by a runtime
-  /// alias check.
-  bool isRuntimeCheckableAliasPair(Instruction *Inst1, Instruction *Inst2);
-
-  /// Records the (distinct base object) pair behind the may-alias dependency
-  /// of \p Inst1 and \p Inst2 as a runtime alias check guarding the region in
-  /// block \p BB. Returns true if the pair was recorded.
-  bool recordRuntimeAliasCheck(BasicBlock *BB, Instruction *Inst1,
-                               Instruction *Inst2);
-
-  /// Emits the collected runtime alias checks and versions the affected block,
-  /// duplicating its body into a scalar fallback guarded by the checks.
-  void versionBlocksForRuntimeChecks();
-
-  /// Builds the i1 value that is true when any pair of checked base objects
-  /// overlaps at runtime. The base address bounds are materialized from their
-  /// SCEVs with \p Exp.
-  Value *emitRuntimeAliasCheck(IRBuilderBase &Builder, SCEVExpander &Exp);
-
-  /// Data to model and emit the runtime alias checks.
-  struct RuntimeAliasCheckInfo {
-    /// The block whose body is guarded by the checks. Exactly one block is
-    /// supported per attempt.
-    BasicBlock *BB = nullptr;
-    /// Pairs of base objects that must be proven disjoint.
-    SmallSetVector<std::pair<const Value *, const Value *>, 4> BasePairs;
-    /// Accessed address range [Low, High) for each involved base object.
-    SmallMapVector<const Value *, std::pair<const SCEV *, const SCEV *>, 4>
-        Bounds;
-
-    void clear() {
-      BB = nullptr;
-      BasePairs.clear();
-      Bounds.clear();
-    }
-  };
-
-  /// When true, scheduling drops may-alias memory dependencies between
-  /// distinct, range-checkable base objects and records them as runtime alias
-  /// checks instead.
-  bool TryRuntimeAliasChecks = false;
-
-  /// Runtime alias checks collected during the last optimistic buildTree().
-  RuntimeAliasCheckInfo RTChecks;
-
-  /// Base-object pairs already proven disjoint by the block's runtime alias
-  /// check.
-  SmallDenseMap<BasicBlock *,
-                SmallDenseSet<std::pair<const Value *, const Value *>, 4>, 2>
-      VersionedBlockCheckedPairs;
-
-  /// Scalar fallback blocks.
-  SmallPtrSet<BasicBlock *, 4> ScalarFallbackBlocks;
-
-  /// Blocks for which a runtime-checks versioning attempt was made
-  /// and did not produce a profitable versioning.
-  SmallPtrSet<BasicBlock *, 8> FailedRuntimeChecksBlocks;
-
-  /// Returns true if a may-alias dependency between the simple load/store
-  /// instructions \p Inst1 and \p Inst2 in block \p BB is already covered by a
-  /// runtime alias check emitted for \p BB by a previous versioning.
-  bool isCoveredByExistingVersionCheck(BasicBlock *BB, Instruction *Inst1,
-                                       Instruction *Inst2) const;
-
-  /// True, if a may-alias dependency between distinct, range-checkable base
-  /// objects is observed (whether or not it was dropped).
-  bool HasRuntimeCheckableBlockers = false;
-
-  /// True, if a kept may-alias dependency is not runtime-checkable (call or a
-  /// non-simple memaccess).
-  bool HasNonCheckableMemBlocker = false;
-
-  /// Runtime checks are validated and bounded the collected checks.
-  bool RTChecksFinalized = false;
-
-  /// Set when a block was versioned with runtime alias checks, which changes
-  /// the CFG. Used to drop CFG-analysis preservation for the run.
-  bool CFGChanged = false;
-
-  /// Guarded block body (non-PHI, non-terminator) in original source order.
-  SmallVector<Instruction *> RTOrigBodyOrder;
-
   using AliasCacheKey = std::pair<Instruction *, Instruction *>;
 
   /// Cache for alias results.
@@ -5714,7 +5517,6 @@ class slpvectorizer::BoUpSLP {
       ScheduleCopyableDataMapByUsers.clear();
       ReadyInsts.clear();
       RecalcCopyableOperandDeps.clear();
-      IgnoredMemDeps.clear();
       ScheduleStart = nullptr;
       ScheduleEnd = nullptr;
       FirstLoadStoreInRegion = nullptr;
@@ -6589,10 +6391,6 @@ class slpvectorizer::BoUpSLP {
     /// recomputing the same operand more than once.
     SmallSetVector<ScheduleData *, 8> RecalcCopyableOperandDeps;
 
-    /// Ordered pairs (Src, Dst) of memory instructions whose may-alias
-    /// dependency has been dropped in favor of a runtime alias check.
-    SmallDenseSet<std::pair<Instruction *, Instruction *>, 8> IgnoredMemDeps;
-
     /// The ID of the scheduling region. For a new vectorization iteration this
     /// is incremented which "removes" all ScheduleData from the region.
     /// Make sure that the initial SchedulingRegionID is greater than the
@@ -24809,457 +24607,6 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
   return nullptr;
 }
 
-bool BoUpSLP::isRuntimeCheckableAliasPair(Instruction *Inst1,
-                                          Instruction *Inst2) {
-  // Only simple (non-volatile, non-atomic) accesses may be reordered once
-  // aliasing is ruled out at runtime; volatile/atomic ordering must be kept.
-  if (!isSimple(Inst1) || !isSimple(Inst2))
-    return false;
-  Value *Ptr1 = getLoadStorePointerOperand(Inst1);
-  Value *Ptr2 = getLoadStorePointerOperand(Inst2);
-  // Only simple load/store accesses have a single pointer operand whose
-  // accessed range can be bounded and compared at runtime.
-  if (!Ptr1 || !Ptr2)
-    return false;
-  if (Ptr1->getType()->getPointerAddressSpace() !=
-      Ptr2->getType()->getPointerAddressSpace())
-    return false;
-  const Value *Base1 = getUnderlyingObject(Ptr1);
-  const Value *Base2 = getUnderlyingObject(Ptr2);
-  // A runtime range check is only meaningful between two distinct, identifiable
-  // objects: two accesses to the same base differ by a compile-time offset, so
-  // they are resolved statically and a base-range check would fold to a
-  // constant predicate rather than a useful runtime guard.
-  if (Base1 == Base2 || isa<UndefValue>(Base1) || isa<UndefValue>(Base2))
-    return false;
-  return true;
-}
-
-bool BoUpSLP::isCoveredByExistingVersionCheck(BasicBlock *BB,
-                                              Instruction *Inst1,
-                                              Instruction *Inst2) const {
-  const auto It = VersionedBlockCheckedPairs.find(BB);
-  if (It == VersionedBlockCheckedPairs.end())
-    return false;
-  Value *Ptr1 = getLoadStorePointerOperand(Inst1);
-  Value *Ptr2 = getLoadStorePointerOperand(Inst2);
-  if (!Ptr1 || !Ptr2)
-    return false;
-  const Value *Base1 = getUnderlyingObject(Ptr1);
-  const Value *Base2 = getUnderlyingObject(Ptr2);
-  if (Base1 == Base2)
-    return false;
-  if (Base2 < Base1)
-    std::swap(Base1, Base2);
-  return It->second.contains({Base1, Base2});
-}
-
-/// Returns true if \p BB's body already contains vector instructions, e.g.
-/// from an earlier SLP vectorization in the same pass.
-static bool blockBodyHasVectorInstructions(BasicBlock *BB) {
-  for (Instruction &I : *BB) {
-    if (isa<PHINode>(&I) || I.isTerminator())
-      continue;
-    // A vector-producing instruction (vector load, binop, shuffle, etc.) has a
-    // vector result type.
-    if (getValueType(&I)->isVectorTy())
-      return true;
-  }
-  return false;
-}
-
-void BoUpSLP::captureRuntimeCheckBodySnapshot() {
-  RTOrigBodyOrder.clear();
-  if (!TryRuntimeAliasChecks || !RTChecks.BB)
-    return;
-  BasicBlock *BB = RTChecks.BB;
-  for (Instruction &I : *BB)
-    if (!isa<PHINode>(&I) && !I.isTerminator())
-      RTOrigBodyOrder.push_back(&I);
-}
-
-bool BoUpSLP::recordRuntimeAliasCheck(BasicBlock *BB, Instruction *Inst1,
-                                      Instruction *Inst2) {
-  Value *Ptr1 = getLoadStorePointerOperand(Inst1);
-  Value *Ptr2 = getLoadStorePointerOperand(Inst2);
-  if (!Ptr1 || !Ptr2)
-    return false;
-  const Value *Base1 = getUnderlyingObject(Ptr1);
-  const Value *Base2 = getUnderlyingObject(Ptr2);
-  if (Base1 == Base2)
-    return false;
-  // Only a single block can be versioned per attempt.
-  if (RTChecks.BB && RTChecks.BB != BB)
-    return false;
-  // Normalize the pair order so duplicate checks collapse.
-  if (Base2 < Base1)
-    std::swap(Base1, Base2);
-  auto Pair = std::make_pair(Base1, Base2);
-  // After the checks have been validated and bounded, do not introduce new
-  // pairs.
-  if (RTChecksFinalized)
-    return RTChecks.BasePairs.contains(Pair);
-  RTChecks.BB = BB;
-  RTChecks.BasePairs.insert(Pair);
-  return true;
-}
-
-bool BoUpSLP::canVersionBlockForRuntimeChecks(BasicBlock *BB) const {
-  assert(BB && "Expected a block to version for runtime checks.");
-  if (!BB->getTerminator())
-    return false;
-  // Versioning duplicates the block body, increasing code size on the guarded
-  // path; do not version when the function is optimized for size (-Os/-Oz).
-  if (F->hasOptSize())
-    return false;
-  if (!DT || !LI)
-    return false;
-  if (!DT->isReachableFromEntry(BB))
-    return false;
-  // Versioning duplicates the block body; only straight-line code outside any
-  // loop is handled for now, to avoid LoopInfo and region updates.
-  if (LI->getLoopFor(BB))
-    return false;
-  // Never version a scalar fallback block: it is the safe, original-order copy
-  // taken when aliasing is detected and must stay scalar.
-  if (ScalarFallbackBlocks.contains(BB))
-    return false;
-  // The scalar fallback must be a faithful copy of the original scalar body.
-  // Reject blocks that were already partially vectorized earlier in this pass.
-  if (blockBodyHasVectorInstructions(BB))
-    return false;
-  // Versioning duplicates the original block body into a scalar fallback.
-  // Calls that cannot be duplicated or whose semantics depend on the call being
-  // immediately followed by a return cannot be cloned into the diamond.
-  if (BB->getTerminatingMustTailCall())
-    return false;
-  if (any_of(*BB, [](Instruction &I) {
-        auto *CB = dyn_cast<CallBase>(&I);
-        return CB && (CB->cannotDuplicate() || CB->isConvergent());
-      }))
-    return false;
-  return true;
-}
-
-bool BoUpSLP::canVersionForRuntimeChecks() {
-  if (RTChecks.BasePairs.empty() || !RTChecks.BB)
-    return false;
-  if (RTChecks.BasePairs.size() > SLPMaxRuntimeAliasChecks)
-    return false;
-  BasicBlock *BB = RTChecks.BB;
-  // Block-level preconditions (loop/optsize/duplicability/...) are the same
-  // ones used to gate the optimistic retry, so reuse them here.
-  if (!canVersionBlockForRuntimeChecks(BB))
-    return false;
-
-  // The scalar fallback is a clone of the body. Operands defined inside the
-  // body are remapped to their clones, but operands defined outside the body
-  // (header PHIs, dominating definitions) are reused as-is by the clone. If
-  // such an outside operand is itself vectorized by this tree, vectorizeTree()
-  // will delete its scalar, leaving the clone with a dangling, out-of-tree use.
-  // Versioning cannot model that, so bail out.
-  for (Instruction &I : *BB) {
-    if (isa<PHINode>(&I) || I.isTerminator())
-      continue;
-    for (Value *Op : I.operands()) {
-      auto *OpI = dyn_cast<Instruction>(Op);
-      if (!OpI)
-        continue;
-      bool DefinedInBody =
-          OpI->getParent() == BB && !isa<PHINode>(OpI) && !OpI->isTerminator();
-      if (!DefinedInBody && isVectorized(OpI))
-        return false;
-    }
-  }
-
-  SmallPtrSet<const Value *, 8> Bases;
-  for (const auto &P : RTChecks.BasePairs) {
-    Bases.insert(P.first);
-    Bases.insert(P.second);
-  }
-  // Every base object must be available in the (PHI-only) header where the
-  // guard branch is emitted.
-  if (any_...
[truncated]